diff options
173 files changed, 2639 insertions, 2995 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java index 3b7d05bf026..706f797cd2c 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java @@ -10,12 +10,15 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.AthenzCertificateClient; +import java.io.IOException; import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.security.cert.Certificate; +import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.time.Duration; -import java.time.Instant; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -34,12 +37,14 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements // TODO Make expiry and update frequency configurable parameters private static final Duration CERTIFICATE_EXPIRY_TIME = Duration.ofDays(30); private static final Duration CERTIFICATE_UPDATE_PERIOD = Duration.ofDays(7); + private static final String DUMMY_PASSWORD = "athenz"; private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); private final AthenzCertificateClient certificateClient; private final KeyProvider keyProvider; private final AthenzProviderServiceConfig.Zones zoneConfig; private final AtomicBoolean alreadyConfigured = new AtomicBoolean(); + private KeyStore initialKeyStore; @Inject public AthenzSslKeyStoreConfigurator(KeyProvider keyProvider, @@ -49,6 +54,7 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements this.certificateClient = new AthenzCertificateClient(config, zoneConfig); this.keyProvider = keyProvider; this.zoneConfig = zoneConfig; + this.initialKeyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig); } @Override @@ -56,8 +62,12 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements if (alreadyConfigured.getAndSet(true)) { // For debugging purpose of SslKeyStoreConfigurator interface throw new IllegalStateException("Already configured. configure() can only be called once."); } - AthenzCertificateUpdater updater = new AthenzCertificateUpdater(sslKeyStoreContext); - scheduler.scheduleAtFixedRate(updater, /*initialDelay*/0, CERTIFICATE_UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); + sslKeyStoreContext.updateKeyStore(initialKeyStore, DUMMY_PASSWORD); + initialKeyStore = null; + scheduler.scheduleAtFixedRate(new AthenzCertificateUpdater(sslKeyStoreContext), + CERTIFICATE_UPDATE_PERIOD.toMinutes()/*initial delay*/, + CERTIFICATE_UPDATE_PERIOD.toMinutes(), + TimeUnit.MINUTES); } @Override @@ -70,6 +80,32 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements } } + private static KeyStore downloadCertificate(KeyProvider keyProvider, + AthenzCertificateClient certificateClient, + AthenzProviderServiceConfig.Zones zoneConfig) { + try { + PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); + X509Certificate certificate = certificateClient.updateCertificate(privateKey, CERTIFICATE_EXPIRY_TIME); + verifyActualExpiry(certificate); + + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null); + keyStore.setKeyEntry("athenz", privateKey, DUMMY_PASSWORD.toCharArray(), new Certificate[]{certificate}); + return keyStore; + } catch (IOException | NoSuchAlgorithmException | CertificateException | KeyStoreException e) { + throw new RuntimeException(e); + } + } + + private static void verifyActualExpiry(X509Certificate certificate) { + Duration actualExpiry = + Duration.between(certificate.getNotBefore().toInstant(), certificate.getNotAfter().toInstant()); + if (CERTIFICATE_EXPIRY_TIME.compareTo(actualExpiry) > 0) { + log.log(LogLevel.WARNING, + String.format("Expected expiry %s, got %s", CERTIFICATE_EXPIRY_TIME, actualExpiry)); + } + } + private class AthenzCertificateUpdater implements Runnable { private final SslKeyStoreContext sslKeyStoreContext; @@ -82,29 +118,13 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements public void run() { try { log.log(LogLevel.INFO, "Updating Athenz certificate from ZTS"); - PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - X509Certificate certificate = certificateClient.updateCertificate(privateKey, CERTIFICATE_EXPIRY_TIME); - verifyActualExperiy(certificate); - - String dummyPassword = "athenz"; - KeyStore keyStore = KeyStore.getInstance("JKS"); - keyStore.load(null); - keyStore.setKeyEntry("athenz", privateKey, dummyPassword.toCharArray(), new Certificate[]{certificate}); - sslKeyStoreContext.updateKeyStore(keyStore, dummyPassword); + KeyStore keyStore = downloadCertificate(keyProvider, certificateClient, zoneConfig); + sslKeyStoreContext.updateKeyStore(keyStore, DUMMY_PASSWORD); log.log(LogLevel.INFO, "Athenz certificate reload successfully completed"); } catch (Throwable e) { log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + e.getMessage(), e); } } - private void verifyActualExperiy(X509Certificate certificate) { - Instant notAfter = certificate.getNotAfter().toInstant(); - Instant notBefore = certificate.getNotBefore().toInstant(); - if (!notBefore.plus(CERTIFICATE_EXPIRY_TIME).equals(notAfter)) { - Duration actualExpiry = Duration.between(notBefore, notAfter); - log.log(LogLevel.WARNING, - String.format("Expected expiry %s, got %s", CERTIFICATE_EXPIRY_TIME, actualExpiry)); - } - } } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSigner.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSigner.java index 2dc3f24664c..1014fc4afdf 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSigner.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSigner.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.ca; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.Zone; import com.yahoo.log.LogLevel; -import com.yahoo.net.HostName; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import org.bouncycastle.asn1.ASN1ObjectIdentifier; @@ -68,13 +68,16 @@ public class CertificateSigner { private final Clock clock; @Inject - public CertificateSigner(KeyProvider keyProvider, AthenzProviderServiceConfig config, Zone zone) { - this(getPrivateKey(keyProvider, config, zone), HostName.getLocalhost(), Clock.systemUTC()); + public CertificateSigner(KeyProvider keyProvider, + ConfigserverConfig configserverConfig, + AthenzProviderServiceConfig config, + Zone zone) { + this(getPrivateKey(keyProvider, config, zone), configserverConfig.loadBalancerAddress(), Clock.systemUTC()); } - CertificateSigner(PrivateKey caPrivateKey, String configServerHostname, Clock clock) { + CertificateSigner(PrivateKey caPrivateKey, String loadBalancerAddress, Clock clock) { this.caPrivateKey = caPrivateKey; - this.issuer = new X500Name("CN=" + configServerHostname); + this.issuer = new X500Name("CN=" + loadBalancerAddress); this.clock = clock; } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 4bf61ff12d8..a69835626ea 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import java.io.File; -import java.net.URI; import java.util.List; import java.util.Optional; import java.util.Set; @@ -47,14 +46,6 @@ public interface ModelContext { boolean hostedVespa(); Zone zone(); Set<Rotation> rotations(); - - /* - * DEPRECATED - * TODO: Remove when 6.172 and earlier are no longer in use - */ - default URI loadBalancerAddress() { - return URI.create("http://localhost"); - } default boolean disableFileDistributor() { return false; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 409e9971b9f..a503fb00825 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -52,6 +52,7 @@ public class VespaMetricSet { private static Set<Metric> getOtherMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("slobrok.heartbeats.failed.count", "slobrok.heartbeats.failed")); + metrics.add(new Metric("logd.processed.lines.count", "logd.processed.lines")); return metrics; } diff --git a/config/src/tests/configagent/configagent.cpp b/config/src/tests/configagent/configagent.cpp index 7eb9442f492..f6593890975 100644 --- a/config/src/tests/configagent/configagent.cpp +++ b/config/src/tests/configagent/configagent.cpp @@ -22,17 +22,17 @@ public: bool abort() override { return false; } bool isAborted() const override { return false; } void setError(int errorCode) override { (void) errorCode; } + bool verifyState(const ConfigState &) const override { return false; } const ConfigKey _key; }; class MyConfigResponse : public ConfigResponse { public: - MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool isUpdated, bool valid, - int64_t timestamp, const vespalib::string & md5, const std::string & errorMsg, int errorC0de, bool iserror) + MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool valid, int64_t timestamp, + const vespalib::string & md5, const std::string & errorMsg, int errorC0de, bool iserror) : _key(key), _value(value), - _isUpdated(isUpdated), _fillCalled(false), _valid(valid), _state(md5, timestamp), @@ -54,7 +54,6 @@ public: const ConfigKey _key; const ConfigValue _value; - bool _isUpdated; bool _fillCalled; bool _valid; const ConfigState _state; @@ -64,24 +63,19 @@ public: Trace _trace; -/** - MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool isUpdated, bool valid, - int64_t timestamp, const vespalib::string & md5, int64_t prevTimestamp, const vespalib::string &prevMd5, - const std::string & errorMsg, int errorC0de, bool iserror) -*/ - static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value) + static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value, uint64_t timestamp = 10, const vespalib::string & md5 = "a") { - return ConfigResponse::UP(new MyConfigResponse(key, value, true, true, 10, "a", "", 0, false)); + return ConfigResponse::UP(new MyConfigResponse(key, value, true, timestamp, md5, "", 0, false)); } static ConfigResponse::UP createServerErrorResponse(const ConfigKey & key, const ConfigValue & value) { - return ConfigResponse::UP(new MyConfigResponse(key, value, false, true, 10, "a", "whinewhine", 2, true)); + return ConfigResponse::UP(new MyConfigResponse(key, value, true, 10, "a", "whinewhine", 2, true)); } static ConfigResponse::UP createConfigErrorResponse(const ConfigKey & key, const ConfigValue & value) { - return ConfigResponse::UP(new MyConfigResponse(key, value, false, false, 10, "a", "", 0, false)); + return ConfigResponse::UP(new MyConfigResponse(key, value, false, 10, "a", "", 0, false)); } }; @@ -154,12 +148,40 @@ TEST("require that successful request is delivered to holder") { handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); ASSERT_TRUE(latch->poll()); ConfigUpdate::UP update(latch->provide()); - ASSERT_TRUE(update.get() != NULL); + ASSERT_TRUE(update); ASSERT_TRUE(update->hasChanged()); MyConfig cfg(update->getValue()); ASSERT_EQUAL("l33t", cfg.myField); } +TEST("require that important(the change) request is delivered to holder even if it was not the last") { + const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); + const ConfigValue testValue1(createValue("l33t", "a")); + const ConfigValue testValue2(createValue("l34t", "b")); + IConfigHolder::SP latch(new MyHolder()); + + FRTConfigAgent handler(latch, testTimingValues); + handler.handleResponse(MyConfigRequest(testKey), + MyConfigResponse::createOKResponse(testKey, testValue1, 1, testValue1.getMd5())); + ASSERT_TRUE(latch->poll()); + ConfigUpdate::UP update(latch->provide()); + ASSERT_TRUE(update); + ASSERT_TRUE(update->hasChanged()); + MyConfig cfg(update->getValue()); + ASSERT_EQUAL("l33t", cfg.myField); + + handler.handleResponse(MyConfigRequest(testKey), + MyConfigResponse::createOKResponse(testKey, testValue2, 2, testValue2.getMd5())); + handler.handleResponse(MyConfigRequest(testKey), + MyConfigResponse::createOKResponse(testKey, testValue2, 3, testValue2.getMd5())); + ASSERT_TRUE(latch->poll()); + update = latch->provide(); + ASSERT_TRUE(update); + ASSERT_TRUE(update->hasChanged()); + MyConfig cfg2(update->getValue()); + ASSERT_EQUAL("l34t", cfg2.myField); +} + TEST("require that successful request sets correct wait time") { const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); const ConfigValue testValue(createValue("l33t", "a")); diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index 1fe91af885b..5f4b7dee215 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -6,10 +6,6 @@ #include <vespa/config/common/configdefinition.h> #include <vespa/config/frt/connection.h> #include <vespa/config/frt/frtsource.h> -#include <vespa/config/frt/frtconfigresponse.h> -#include <vespa/config/frt/frtconfigrequest.h> -#include <vespa/config/frt/frtconfigrequestv2.h> -#include <vespa/config/frt/frtconfigresponsev2.h> #include <vespa/config/frt/frtconfigrequestv3.h> #include <vespa/config/frt/frtconfigresponsev3.h> #include <vespa/vespalib/data/slime/slime.h> @@ -232,107 +228,27 @@ namespace { TEST_F("require that empty config response does not validate", RPCFixture()) { - FRTConfigResponseV1 fail1(f1.createEmptyRequest()); + FRTConfigResponseV3 fail1(f1.createEmptyRequest()); ASSERT_FALSE(fail1.validateResponse()); ASSERT_FALSE(fail1.hasValidResponse()); ASSERT_TRUE(fail1.isError()); } TEST_F("require that response containing errors does not validate", RPCFixture()) { - FRTConfigResponseV1 fail1(f1.createErrorRequest()); + FRTConfigResponseV3 fail1(f1.createErrorRequest()); ASSERT_FALSE(fail1.validateResponse()); ASSERT_FALSE(fail1.hasValidResponse()); ASSERT_TRUE(fail1.isError()); ASSERT_TRUE(fail1.errorCode() != 0); } -TEST_F("require that valid response validates", RPCFixture()) { - std::vector<vespalib::string> vec; - vec.push_back("bar \"foo\""); - FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15, vec, "mn")); - ASSERT_TRUE(ok.validateResponse()); - ASSERT_TRUE(ok.hasValidResponse()); -} - TEST_F("require that response contains all values", RPCFixture()) { - FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15)); + FRTConfigResponseV3 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15)); ASSERT_FALSE(ok.validateResponse()); ASSERT_FALSE(ok.hasValidResponse()); } -TEST_F("require that valid response returns values after fill", RPCFixture()) { - std::vector<vespalib::string> vec; - vec.push_back("bar \"foo\""); - FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15, vec, "mn")); - ASSERT_TRUE(ok.validateResponse()); - ASSERT_TRUE(ok.hasValidResponse()); - - // Should not be valid - ASSERT_TRUE(ConfigKey() == ok.getKey()); - ASSERT_TRUE(ConfigValue() == ok.getValue()); - - ok.fill(); - ConfigKey key(ok.getKey()); - ASSERT_EQUAL("foo", key.getDefName()); - ASSERT_EQUAL("baz", key.getDefMd5()); - ASSERT_EQUAL("bim", key.getConfigId()); - ASSERT_EQUAL("mn", key.getDefNamespace()); - - ConfigValue value(ok.getValue()); - ASSERT_TRUE(vec == value.getLines()); -} - -TEST("require that request parameters are correctly initialized") { - ConnectionMock conn; - std::vector<vespalib::string> schema; - schema.push_back("foo"); - schema.push_back("bar"); - ConfigKey key("foo", "bar", "bim", "boo", schema); - FRTConfigRequestV1 frtReq(key, &conn, "mymd5", 1337, 8); - - FRT_RPCRequest * req = frtReq.getRequest(); - FRT_Values & params(*req->GetParams()); - ASSERT_EQUAL("bar", std::string(params[0]._string._str)); - ASSERT_EQUAL("", std::string(params[1]._string._str)); - ASSERT_EQUAL("boo", std::string(params[2]._string._str)); - ASSERT_EQUAL("foo", std::string(params[3]._string._str)); - ASSERT_EQUAL("mymd5", std::string(params[4]._string._str)); - ASSERT_EQUAL(1337u, params[5]._intval64); - ASSERT_EQUAL(8u, params[6]._intval64); - ASSERT_EQUAL("bim", std::string(params[7]._string._str)); - ASSERT_EQUAL(2u, params[8]._string_array._len); - ASSERT_EQUAL("foo", std::string(params[8]._string_array._pt[0]._str)); - ASSERT_EQUAL("bar", std::string(params[8]._string_array._pt[1]._str)); - ASSERT_EQUAL(1u, params[9]._intval32); -} - -TEST("require that request is aborted") { - MyAbortHandler handler; - ConnectionMock conn; - ConfigKey key("foo", "bar", "bim", "boo"); - FRTConfigRequestV1 frtReq(key, &conn, "mymd5", 1337, 8); - frtReq.getRequest()->SetAbortHandler(&handler); - ASSERT_FALSE(frtReq.isAborted()); - ASSERT_TRUE(frtReq.abort()); -} - -TEST_FF("require that request is invoked", SourceFixture(), - FRTFixture(f1)) -{ - f2.result.state = ConfigState("foo", 3); - f2.src.getConfig(); - ASSERT_TRUE(f2.src.getCurrentRequest().verifyKey(f1.key)); - ASSERT_FALSE(f2.src.getCurrentRequest().verifyKey(ConfigKey("foo", "bal", "bim", "boo", std::vector<vespalib::string>()))); - ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 0))); - ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 1))); - ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("bar", 1))); - ASSERT_TRUE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 3))); - ASSERT_TRUE(f2.result.notified); - f2.src.close(); -} - -TEST_FF("require that request is config task is scheduled", SourceFixture(), - FRTFixture(f1)) +TEST_FF("require that request is config task is scheduled", SourceFixture(), FRTFixture(f1)) { f2.src.getConfig(); ASSERT_TRUE(f2.result.notified); @@ -349,96 +265,6 @@ TEST_FF("require that request is config task is scheduled", SourceFixture(), f2.src.close(); } -TEST("require that v2 request is correctly initialized") { - ConnectionMock conn; - ConfigKey key = ConfigKey::create<MyConfig>("foobi"); - vespalib::string md5 = "mymd5"; - int64_t currentGeneration = 3; - int64_t wantedGeneration = 4; - vespalib::string hostName = "myhost"; - int64_t timeout = 3000; - Trace traceIn(3); - traceIn.trace(2, "Hei"); - FRTConfigRequestV2 v2req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, timeout, traceIn); - ConfigDefinition origDef(MyConfig::CONFIG_DEF_SCHEMA); - - FRT_RPCRequest * req = v2req.getRequest(); - ASSERT_TRUE(req != NULL); - FRT_Values & params(*req->GetParams()); - std::string json(params[0]._string._str); - Slime slime; - JsonFormat::decode(Memory(json), slime); - Inspector & root(slime.get()); - EXPECT_EQUAL(2, root[REQUEST_VERSION].asLong()); - EXPECT_EQUAL(key.getDefName(), root[REQUEST_DEF_NAME].asString().make_string()); - EXPECT_EQUAL(key.getDefNamespace(), root[REQUEST_DEF_NAMESPACE].asString().make_string()); - EXPECT_EQUAL(key.getDefMd5(), root[REQUEST_DEF_MD5].asString().make_string()); - EXPECT_EQUAL(key.getConfigId(), root[REQUEST_CLIENT_CONFIGID].asString().make_string()); - EXPECT_EQUAL(hostName, root[REQUEST_CLIENT_HOSTNAME].asString().make_string()); - EXPECT_EQUAL(currentGeneration, root[REQUEST_CURRENT_GENERATION].asLong()); - EXPECT_EQUAL(wantedGeneration, root[REQUEST_WANTED_GENERATION].asLong()); - EXPECT_EQUAL(md5, root[REQUEST_CONFIG_MD5].asString().make_string()); - EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); - Trace trace; - trace.deserialize(root[REQUEST_TRACE]); - EXPECT_TRUE(trace.shouldTrace(2)); - EXPECT_TRUE(trace.shouldTrace(3)); - EXPECT_FALSE(trace.shouldTrace(4)); - EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); - ConfigDefinition def; - def.deserialize(root[REQUEST_DEF_CONTENT]); - EXPECT_EQUAL(origDef.asString(), def.asString()); - ConfigResponse::UP response(v2req.createResponse(req)); - req->GetReturn()->AddString("foobar"); - EXPECT_TRUE(response->validateResponse()); -} - -TEST("require that v2 reponse is correctly initialized") { - ConnectionMock conn; - Slime slime; - ConfigKey key = ConfigKey::create<MyConfig>("foobi"); - vespalib::string md5 = "mymd5"; - int64_t generation = 3; - vespalib::string hostname = "myhhost"; - Trace traceIn(3); - traceIn.trace(2, "Hei!"); - Cursor & root(slime.setObject()); - root.setLong(RESPONSE_VERSION, 2ul); - root.setString(RESPONSE_DEF_NAME, Memory(key.getDefName())); - root.setString(RESPONSE_DEF_NAMESPACE, Memory(key.getDefNamespace())); - root.setString(RESPONSE_DEF_MD5, Memory(key.getDefMd5())); - root.setString(RESPONSE_CONFIGID, Memory(key.getConfigId())); - root.setString(RESPONSE_CLIENT_HOSTNAME, Memory(hostname)); - root.setString(RESPONSE_CONFIG_MD5, Memory(md5)); - root.setLong(RESPONSE_CONFIG_GENERATION, generation); - traceIn.serialize(root.setObject(RESPONSE_TRACE)); - Cursor & payload(root.setObject(RESPONSE_PAYLOAD)); - payload.setString("myField", "foobiar"); - SimpleBuffer buf; - JsonFormat::encode(slime, buf, true); - FRT_RPCRequest * req = conn.allocRPCRequest(); - req->GetReturn()->AddString(buf.get().make_string().c_str()); - FRTConfigResponseV2 response(req); - ASSERT_TRUE(response.validateResponse()); - response.fill(); - Trace trace(response.getTrace()); - EXPECT_TRUE(trace.shouldTrace(3)); - EXPECT_FALSE(trace.shouldTrace(4)); - ConfigKey responseKey(response.getKey()); - EXPECT_EQUAL(key.getDefName(), responseKey.getDefName()); - EXPECT_EQUAL(key.getDefNamespace(), responseKey.getDefNamespace()); - EXPECT_EQUAL(key.getDefMd5(), responseKey.getDefMd5()); - EXPECT_EQUAL(key.getConfigId(), responseKey.getConfigId()); - EXPECT_EQUAL(hostname, response.getHostName()); - ConfigState state(response.getConfigState()); - EXPECT_EQUAL(md5, state.md5); - EXPECT_EQUAL(generation, state.generation); - ConfigValue value(response.getValue()); - MyConfig::UP config(value.newInstance<MyConfig>()); - EXPECT_EQUAL("foobiar", config->myField); - req->SubRef(); -} - TEST("require that v3 request is correctly initialized") { ConnectionMock conn; ConfigKey key = ConfigKey::create<MyConfig>("foobi"); @@ -449,7 +275,13 @@ TEST("require that v3 request is correctly initialized") { int64_t timeout = 3000; Trace traceIn(3); traceIn.trace(2, "Hei"); - FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); + FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, + timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); + ASSERT_TRUE(v3req.verifyState(ConfigState(md5, 3))); + ASSERT_FALSE(v3req.verifyState(ConfigState(md5, 2))); + ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 3))); + ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 2))); + ConfigDefinition origDef(MyConfig::CONFIG_DEF_SCHEMA); FRT_RPCRequest * req = v3req.getRequest(); diff --git a/config/src/vespa/config/common/configholder.cpp b/config/src/vespa/config/common/configholder.cpp index e94510dcdea..6eeb1bd79a2 100644 --- a/config/src/vespa/config/common/configholder.cpp +++ b/config/src/vespa/config/common/configholder.cpp @@ -39,7 +39,7 @@ bool ConfigHolder::poll() { vespalib::MonitorGuard guard(_monitor); - return (_current.get() != NULL); + return static_cast<bool>(_current); } void diff --git a/config/src/vespa/config/common/configrequest.h b/config/src/vespa/config/common/configrequest.h index 395b09abcb0..efe1e56cdd7 100644 --- a/config/src/vespa/config/common/configrequest.h +++ b/config/src/vespa/config/common/configrequest.h @@ -24,15 +24,13 @@ public: ConfigRequest() { } virtual ~ConfigRequest() { } - virtual const ConfigKey & getKey() const = 0; - /** Abort a request. */ virtual bool abort() = 0; - virtual bool isAborted() const = 0; - virtual void setError(int errorCode) = 0; + virtual bool verifyState(const ConfigState & state) const = 0; + }; } diff --git a/config/src/vespa/config/frt/CMakeLists.txt b/config/src/vespa/config/frt/CMakeLists.txt index 9d0966bb09a..879b32684a7 100644 --- a/config/src/vespa/config/frt/CMakeLists.txt +++ b/config/src/vespa/config/frt/CMakeLists.txt @@ -8,9 +8,7 @@ vespa_add_library(config_frt OBJECT frtconfigresponse.cpp frtsourcefactory.cpp frtconfigagent.cpp - frtconfigrequestv2.cpp frtconfigrequestfactory.cpp - frtconfigresponsev2.cpp protocol.cpp slimeconfigrequest.cpp slimeconfigresponse.cpp diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp index ff16ef77a1b..f475cff0a9a 100644 --- a/config/src/vespa/config/frt/frtconfigagent.cpp +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigagent.h" +#include "frtconfigrequestv3.h" #include <vespa/config/common/trace.h> #include <vespa/log/log.h> @@ -29,14 +30,14 @@ FRTConfigAgent::handleResponse(const ConfigRequest & request, ConfigResponse::UP LOG(spam, "current state for %s: generation %ld md5 %s", key.toString().c_str(), _configState.generation, _configState.md5.c_str()); } if (response->validateResponse() && !response->isError()) { - handleOKResponse(std::move(response)); + handleOKResponse(request, std::move(response)); } else { handleErrorResponse(request, std::move(response)); } } void -FRTConfigAgent::handleOKResponse(ConfigResponse::UP response) +FRTConfigAgent::handleOKResponse(const ConfigRequest & request, ConfigResponse::UP response) { _failedRequests = 0; response->fill(); @@ -45,8 +46,7 @@ FRTConfigAgent::handleOKResponse(ConfigResponse::UP response) } ConfigState newState = response->getConfigState(); - bool isNewGeneration = newState.isNewerGenerationThan(_configState); - if (isNewGeneration) { + if ( ! request.verifyState(newState)) { handleUpdatedGeneration(response->getKey(), newState, response->getValue()); } setWaitTime(_timingValues.successDelay, 1); @@ -57,22 +57,20 @@ void FRTConfigAgent::handleUpdatedGeneration(const ConfigKey & key, const ConfigState & newState, const ConfigValue & configValue) { if (LOG_WOULD_LOG(spam)) { - LOG(spam, "new generation %ld for key %s", newState.generation, key.toString().c_str()); + LOG(spam, "new generation %ld md5:%s for key %s", newState.generation, newState.md5.c_str(), key.toString().c_str()); + LOG(spam, "Old config: md5:%s \n%s", _latest.getMd5().c_str(), _latest.asJson().c_str()); + LOG(spam, "New config: md5:%s \n%s", configValue.getMd5().c_str(), configValue.asJson().c_str()); } - _configState.generation = newState.generation; - bool hasDifferentPayload = newState.hasDifferentPayloadFrom(_configState); - if (hasDifferentPayload) { - if (LOG_WOULD_LOG(spam)) { - LOG(spam, "new payload for key %s, existing md5(%s), new md5(%s)", key.toString().c_str(), _configState.md5.c_str(), newState.md5.c_str()); - } - _configState.md5 = newState.md5; + if (_latest.getMd5() != configValue.getMd5()) { _latest = configValue; } + _configState = newState; + if (LOG_WOULD_LOG(spam)) { - LOG(spam, "updating holder for key %s, payload changed: %d", key.toString().c_str(), hasDifferentPayload ? 1 : 0); + LOG(spam, "updating holder for key %s,", key.toString().c_str()); } - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(_latest, hasDifferentPayload, _configState.generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(_latest, true, newState.generation))); _numConfigured++; } diff --git a/config/src/vespa/config/frt/frtconfigagent.h b/config/src/vespa/config/frt/frtconfigagent.h index e3c362b5278..97edefbded7 100644 --- a/config/src/vespa/config/frt/frtconfigagent.h +++ b/config/src/vespa/config/frt/frtconfigagent.h @@ -33,7 +33,7 @@ public: const ConfigState & getConfigState() const override; private: void handleUpdatedGeneration(const ConfigKey & key, const ConfigState & newState, const ConfigValue & configValue); - void handleOKResponse(ConfigResponse::UP response); + void handleOKResponse(const ConfigRequest & request, ConfigResponse::UP response); void handleErrorResponse(const ConfigRequest & request, ConfigResponse::UP response); void setWaitTime(uint64_t delay, int multiplier); diff --git a/config/src/vespa/config/frt/frtconfigrequest.cpp b/config/src/vespa/config/frt/frtconfigrequest.cpp index aa792126aca..0845f7408de 100644 --- a/config/src/vespa/config/frt/frtconfigrequest.cpp +++ b/config/src/vespa/config/frt/frtconfigrequest.cpp @@ -2,9 +2,7 @@ #include "frtconfigrequest.h" #include "frtconfigresponse.h" #include "connection.h" -#include <vespa/fnet/frt/frt.h> -#include <vespa/config/common/configkey.h> -#include <vespa/config/common/configstate.h> +#include <vespa/fnet/frt/rpcrequest.h> namespace config { @@ -45,52 +43,4 @@ FRTConfigRequest::isAborted() const return (_request->GetErrorCode() == FRTE_RPC_ABORT); } -const vespalib::string FRTConfigRequestV1::REQUEST_TYPES = "sssssllsSi"; - -FRTConfigRequestV1::FRTConfigRequestV1(const ConfigKey & key, - Connection * connection, - const vespalib::string & configMd5, - int64_t generation, - int64_t serverTimeout) - : FRTConfigRequest(connection, key) -{ - _request->SetMethodName("config.v1.getConfig"); - _parameters.AddString(key.getDefName().c_str()); - _parameters.AddString(""); - _parameters.AddString(key.getDefMd5().c_str()); - _parameters.AddString(key.getConfigId().c_str()); - _parameters.AddString(configMd5.c_str()); - _parameters.AddInt64(generation); - _parameters.AddInt64(serverTimeout); - _parameters.AddString(key.getDefNamespace().c_str()); - const std::vector<vespalib::string> & schema(key.getDefSchema()); - FRT_StringValue * schemaValue = _parameters.AddStringArray(schema.size()); - for (size_t i = 0; i < schema.size(); i++) { - _parameters.SetString(&schemaValue[i], schema[i].c_str()); - } - _parameters.AddInt32(1); -} - -bool -FRTConfigRequestV1::verifyKey(const ConfigKey & key) const -{ - return (key.getDefName().compare(_parameters[0]._string._str) == 0 && - key.getDefNamespace().compare(_parameters[7]._string._str) == 0 && - key.getConfigId().compare(_parameters[3]._string._str) == 0 && - key.getDefMd5().compare(_parameters[2]._string._str) == 0); -} - -bool -FRTConfigRequestV1::verifyState(const ConfigState & state) const -{ - return (state.md5.compare(_parameters[4]._string._str) == 0 && - state.generation == static_cast<int64_t>(_parameters[5]._intval64)); -} - -ConfigResponse::UP -FRTConfigRequestV1::createResponse(FRT_RPCRequest * request) const -{ - return ConfigResponse::UP(new FRTConfigResponseV1(request)); -} - } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigrequest.h b/config/src/vespa/config/frt/frtconfigrequest.h index e1d6b5590bd..061151d5f39 100644 --- a/config/src/vespa/config/frt/frtconfigrequest.h +++ b/config/src/vespa/config/frt/frtconfigrequest.h @@ -22,8 +22,6 @@ public: typedef std::unique_ptr<FRTConfigRequest> UP; FRTConfigRequest(Connection * connection, const ConfigKey & key); ~FRTConfigRequest(); - virtual bool verifyKey(const ConfigKey & key) const = 0; - virtual bool verifyState(const ConfigState & state) const = 0; bool abort() override; bool isAborted() const override; @@ -33,25 +31,11 @@ public: FRT_RPCRequest* getRequest() { return _request; } virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const = 0; protected: - FRT_RPCRequest *_request; - FRT_Values & _parameters; + FRT_RPCRequest * _request; + FRT_Values & _parameters; private: - Connection * _connection; - const ConfigKey _key; -}; - -class FRTConfigRequestV1 : public FRTConfigRequest { -public: - FRTConfigRequestV1(const ConfigKey & key, - Connection * connection, - const vespalib::string & configMd5, - int64_t generation, - int64_t serverTimeout); - bool verifyKey(const ConfigKey & key) const override; - bool verifyState(const ConfigState & state) const override; - ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override; -private: - static const vespalib::string REQUEST_TYPES; + Connection * _connection; + const ConfigKey _key; }; } diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp index 9fe3e073b65..1f1ddb196b0 100644 --- a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp @@ -1,14 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigrequestfactory.h" -#include "frtconfigrequest.h" -#include "frtconfigrequestv2.h" #include "frtconfigrequestv3.h" -#include <vespa/config/common/trace.h> -#include <vespa/config/common/compressiontype.h> #include <vespa/vespalib/util/host_name.h> -#include <unistd.h> -#include <limits.h> +using std::make_unique; namespace config { @@ -28,14 +23,11 @@ FRTConfigRequestFactory::~FRTConfigRequestFactory() { } FRTConfigRequest::UP -FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const +FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, + const ConfigState & state, int64_t serverTimeout) const { - if (1 == _protocolVersion) { - return FRTConfigRequest::UP(new FRTConfigRequestV1(key, connection, state.md5, state.generation, serverTimeout)); - } else if (2 == _protocolVersion) { - return FRTConfigRequest::UP(new FRTConfigRequestV2(connection, key, state.md5, state.generation, 0u, _hostName, serverTimeout, Trace(_traceLevel))); - } - return FRTConfigRequest::UP(new FRTConfigRequestV3(connection, key, state.md5, state.generation, 0u, _hostName, serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType)); + return make_unique<FRTConfigRequestV3>(connection, key, state.md5, state.generation, 0u, _hostName, + serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType); } } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigrequestv2.cpp b/config/src/vespa/config/frt/frtconfigrequestv2.cpp deleted file mode 100644 index ddc79b830b3..00000000000 --- a/config/src/vespa/config/frt/frtconfigrequestv2.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "frtconfigrequestv2.h" -#include "frtconfigresponsev2.h" -#include "connection.h" -#include <vespa/config/common/trace.h> -#include <vespa/config/common/vespa_version.h> - -using namespace config::protocol; - -namespace config { - -FRTConfigRequestV2::FRTConfigRequestV2(Connection * connection, - const ConfigKey & key, - const vespalib::string & configMd5, - int64_t currentGeneration, - int64_t wantedGeneration, - const vespalib::string & hostName, - int64_t serverTimeout, - const Trace & trace) - : SlimeConfigRequest(connection, key, configMd5, currentGeneration, wantedGeneration, hostName, serverTimeout, trace, VespaVersion::getCurrentVersion(), 2, CompressionType::UNCOMPRESSED, "config.v2.getConfig") -{ -} - - - -ConfigResponse::UP -FRTConfigRequestV2::createResponse(FRT_RPCRequest * request) const -{ - return ConfigResponse::UP(new FRTConfigResponseV2(request)); -} - -} diff --git a/config/src/vespa/config/frt/frtconfigrequestv2.h b/config/src/vespa/config/frt/frtconfigrequestv2.h deleted file mode 100644 index 5f055153b8c..00000000000 --- a/config/src/vespa/config/frt/frtconfigrequestv2.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "slimeconfigrequest.h" - -class FRT_Values; -class FRT_RPCRequest; - -namespace config { - -class ConfigKey; -class Connection; -class Trace; - -class FRTConfigRequestV2 : public SlimeConfigRequest { -public: - FRTConfigRequestV2(Connection * connection, - const ConfigKey & key, - const vespalib::string & configMd5, - int64_t currentGeneration, - int64_t wantedGeneration, - const vespalib::string & hostName, - int64_t serverTimeout, - const Trace & trace); - ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override; -}; - -} - diff --git a/config/src/vespa/config/frt/frtconfigresponse.cpp b/config/src/vespa/config/frt/frtconfigresponse.cpp index 98922ad4729..046416bcfd6 100644 --- a/config/src/vespa/config/frt/frtconfigresponse.cpp +++ b/config/src/vespa/config/frt/frtconfigresponse.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigresponse.h" -#include <vespa/fnet/frt/frt.h> +#include <vespa/fnet/frt/rpcrequest.h> namespace config { @@ -42,52 +42,4 @@ vespalib::string FRTConfigResponse::errorMessage() const { return _request->GetE int FRTConfigResponse::errorCode() const { return _request->GetErrorCode(); } bool FRTConfigResponse::isError() const { return _request->IsError(); } -// -// V1 Implementation -// -const vespalib::string FRTConfigResponseV1::RESPONSE_TYPES = "sssssilSs"; - -FRTConfigResponseV1::FRTConfigResponseV1(FRT_RPCRequest * request) - : FRTConfigResponse(request), - _key(), - _value() -{ -} - -FRTConfigResponseV1::~FRTConfigResponseV1() {} - -const vespalib::string & -FRTConfigResponseV1::getResponseTypes() const -{ - return RESPONSE_TYPES; -} - -void -FRTConfigResponseV1::fill() -{ - const std::vector<vespalib::string> payload(getPayLoad()); - _value = ConfigValue(payload, calculateContentMd5(payload)); - _key = readKey(); - _state = ConfigState(vespalib::string((*_returnValues)[4]._string._str), (*_returnValues)[6]._intval64); -} - -const ConfigKey -FRTConfigResponseV1::readKey() const -{ - return ConfigKey((*_returnValues)[3]._string._str, (*_returnValues)[0]._string._str, (*_returnValues)[8]._string._str, (*_returnValues)[2]._string._str); -} - -const std::vector<vespalib::string> -FRTConfigResponseV1::getPayLoad() const -{ - uint32_t numStrings = (*_returnValues)[7]._string_array._len; - FRT_StringValue *s = (*_returnValues)[7]._string_array._pt; - std::vector<vespalib::string> payload; - payload.reserve(numStrings); - for (uint32_t i = 0; i < numStrings; i++) { - payload.push_back(vespalib::string(s[i]._str)); - } - return payload; -} - } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponse.h b/config/src/vespa/config/frt/frtconfigresponse.h index f829044e698..31811c6a38f 100644 --- a/config/src/vespa/config/frt/frtconfigresponse.h +++ b/config/src/vespa/config/frt/frtconfigresponse.h @@ -39,33 +39,5 @@ protected: FRT_Values * _returnValues; }; -class FRTConfigResponseV1 : public FRTConfigResponse { -private: - FRTConfigResponseV1& operator=(const FRTConfigResponseV1&); -public: - FRTConfigResponseV1(FRT_RPCRequest * request); - ~FRTConfigResponseV1(); - - const ConfigKey & getKey() const override { return _key; } - const ConfigValue & getValue() const override { return _value; } - const Trace & getTrace() const override { return _trace; } - - const ConfigState & getConfigState() const override { return _state; } - - void fill() override; - -private: - static const vespalib::string RESPONSE_TYPES; - - const std::vector<vespalib::string> getPayLoad() const; - const ConfigKey readKey() const; - const vespalib::string & getResponseTypes() const override; - - ConfigKey _key; - ConfigValue _value; - ConfigState _state; - Trace _trace; -}; - } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponsev2.cpp b/config/src/vespa/config/frt/frtconfigresponsev2.cpp deleted file mode 100644 index 1527700f99f..00000000000 --- a/config/src/vespa/config/frt/frtconfigresponsev2.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "frtconfigresponsev2.h" -#include <vespa/fnet/frt/frt.h> - -using namespace vespalib; -using namespace vespalib::slime; -using namespace vespalib::slime::convenience; -using namespace config::protocol::v2; - -namespace config { - -class V2Payload : public protocol::Payload { -public: - V2Payload(const SlimePtr & data) - : _data(data) - {} - const Inspector & getSlimePayload() const override { - return extractPayload(*_data); - } -private: - SlimePtr _data; -}; - -const vespalib::string FRTConfigResponseV2::RESPONSE_TYPES = "s"; - -FRTConfigResponseV2::FRTConfigResponseV2(FRT_RPCRequest * request) - : SlimeConfigResponse(request) -{ -} - -const vespalib::string & -FRTConfigResponseV2::getResponseTypes() const -{ - return RESPONSE_TYPES; -} - -const ConfigValue -FRTConfigResponseV2::readConfigValue() const -{ - vespalib::string md5(_data->get()[RESPONSE_CONFIG_MD5].asString().make_string()); - return ConfigValue(PayloadPtr(new V2Payload(_data)), md5); -} - -} // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponsev2.h b/config/src/vespa/config/frt/frtconfigresponsev2.h deleted file mode 100644 index 89d1a9157e3..00000000000 --- a/config/src/vespa/config/frt/frtconfigresponsev2.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "slimeconfigresponse.h" -#include <vespa/config/common/configvalue.h> - -class FRT_RPCRequest; -class FRT_Values; - -namespace config { - -/** - * Baseclass for config responses. - */ -class FRTConfigResponseV2 : public SlimeConfigResponse { -private: - FRTConfigResponseV2& operator=(const FRTConfigResponseV2&); -public: - FRTConfigResponseV2(FRT_RPCRequest * request); - -private: - static const vespalib::string RESPONSE_TYPES; - const vespalib::string & getResponseTypes() const override; - const ConfigValue readConfigValue() const override; -}; - -} // namespace config - diff --git a/config/src/vespa/config/frt/slimeconfigrequest.cpp b/config/src/vespa/config/frt/slimeconfigrequest.cpp index 12dbbab2eb7..27ac39ae56a 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.cpp +++ b/config/src/vespa/config/frt/slimeconfigrequest.cpp @@ -40,19 +40,10 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, } bool -SlimeConfigRequest::verifyKey(const ConfigKey & key) const -{ - return (key.getDefName().compare(_parameters[0]._string._str) == 0 && - key.getDefNamespace().compare(_parameters[7]._string._str) == 0 && - key.getConfigId().compare(_parameters[3]._string._str) == 0 && - key.getDefMd5().compare(_parameters[2]._string._str) == 0); -} - -bool SlimeConfigRequest::verifyState(const ConfigState & state) const { - return (state.md5.compare(_parameters[4]._string._str) == 0 && - state.generation == static_cast<int64_t>(_parameters[5]._intval64)); + return (state.md5.compare(_data[REQUEST_CONFIG_MD5].asString().make_stringref()) == 0 && + state.generation == _data[REQUEST_CURRENT_GENERATION].asLong()); } void diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h index d616876780e..625d70c094b 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.h +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -30,7 +30,6 @@ public: const CompressionType & compressionType, const vespalib::string & methodName); ~SlimeConfigRequest() {} - bool verifyKey(const ConfigKey & key) const override; bool verifyState(const ConfigState & state) const override; virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override = 0; private: diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index d8a560c6159..e991341b616 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -35,7 +35,7 @@ public class FileDirectory { try { ensureRootExist(); } catch (IllegalArgumentException e) { - log.warning("Failed creating directory in constructor, will retry on demand : " + e.toString()); + log.log(LogLevel.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.toString()); } } @@ -70,12 +70,8 @@ public class FileDirectory { throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + "' is not a directory."); } File [] files = dir.listFiles(new Filter()); - if (files.length != 1) { - StringBuilder msg = new StringBuilder(); - for (File f: files) { - msg.append(f.getName()).append("\n"); - } - throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain exactly one file, but [" + msg.toString() + "]"); + if (files == null || files.length == 0) { + throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain any files"); } return files[0]; } @@ -96,25 +92,28 @@ public class FileDirectory { } } - public FileReference addFile(File source, FileReference reference) { + FileReference addFile(File source, FileReference reference) { ensureRootExist(); try { logfileInfo(source); File destinationDir = new File(root, reference.value()); + Path tempDestinationDir = Files.createTempDirectory(root.toPath(), "writing"); + File destination = new File(tempDestinationDir.toFile(), source.getName()); if (!destinationDir.exists()) { destinationDir.mkdir(); - Path tempDestinationDir = Files.createTempDirectory(root.toPath(), "writing"); - File destination = new File(tempDestinationDir.toFile(), source.getName()); - if (source.isDirectory()) - IOUtils.copyDirectory(source, destination); - else + log.log(LogLevel.DEBUG, "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() ); + if (source.isDirectory()) { + log.log(LogLevel.DEBUG, "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath()); + IOUtils.copyDirectory(source, destination, -1); + } else copyFile(source, destination); if (!destinationDir.exists()) { + log.log(LogLevel.DEBUG, "Moving from " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) { - log.warning("Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); + log.log(LogLevel.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); } } else { - IOUtils.copyDirectory(tempDestinationDir.toFile(), destinationDir, 1); + IOUtils.copyDirectory(tempDestinationDir.toFile(), destinationDir, -1); } IOUtils.recursiveDeleteDir(tempDestinationDir.toFile()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 9316a9a5c8e..958f26632ef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -14,7 +14,9 @@ import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.filedistribution.CompressedFileReference; import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.vespa.filedistribution.FileReferenceData; import java.io.File; import java.io.IOException; @@ -26,6 +28,7 @@ import java.util.stream.Collectors; public class FileServer { private static final Logger log = Logger.getLogger(FileServer.class.getName()); + private final FileDirectory root; private final ExecutorService executor; private final FileDownloader downloader; @@ -43,7 +46,7 @@ public class FileServer { } public interface Receiver { - void receive(FileReference reference, String filename, byte [] content, ReplayStatus status); + void receive(FileReferenceData fileData, ReplayStatus status); } @Inject @@ -87,22 +90,39 @@ public class FileServer { File file = root.getFile(reference); // TODO remove once verified in system tests. log.info("Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); - byte [] blob = new byte [0]; boolean success = false; String errorDescription = "OK"; + FileReferenceData fileData = FileReferenceData.empty(reference, file.getName()); try { - blob = IOUtils.readFileBytes(file); + fileData = readFileReferenceData(reference); success = true; } catch (IOException e) { - errorDescription = "For file reference '" + reference.value() + "' I failed reading file '" + file.getAbsolutePath() + "'"; - log.warning(errorDescription + "for sending to '" + target.toString() + "'. " + e.toString()); + errorDescription = "For file reference '" + reference.value() + "': failed reading file '" + file.getAbsolutePath() + "'"; + log.warning(errorDescription + " for sending to '" + target.toString() + "'. " + e.toString()); } - target.receive(reference, file.getName(), blob, - new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); + + target.receive(fileData, new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); // TODO remove once verified in system tests. log.info("Done serving reference '" + reference.toString() + "' with file '" + file.getAbsolutePath() + "'"); } + + private FileReferenceData readFileReferenceData(FileReference reference) throws IOException { + File file = root.getFile(reference); + + byte[] blob; + FileReferenceData.Type type; + if (file.isDirectory()) { + type = FileReferenceData.Type.compressed; + blob = CompressedFileReference.compress(file.getParentFile()); + } else { + type = FileReferenceData.Type.file; + blob = IOUtils.readFileBytes(file); + } + + return new FileReferenceData(reference, file.getName(), type, blob); + } + public void download(FileReference fileReference) { downloader.getFile(fileReference); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index d17cdf722ea..5c50fbfc31b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -41,10 +41,8 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdaterFactory; import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; import com.yahoo.vespa.config.server.tenant.TenantListener; import com.yahoo.vespa.config.server.tenant.Tenants; -import net.jpountz.xxhash.XXHash64; -import net.jpountz.xxhash.XXHashFactory; +import com.yahoo.vespa.filedistribution.FileReferenceData; -import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -438,19 +436,19 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } @Override - public void receive(FileReference reference, String filename, byte [] content, FileServer.ReplayStatus status) { - XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + public void receive(FileReferenceData fileData, FileServer.ReplayStatus status) { Request fileBlob = new Request("filedistribution.receiveFile"); - fileBlob.parameters().add(new StringValue(reference.value())); - fileBlob.parameters().add(new StringValue(filename)); - fileBlob.parameters().add(new DataValue(content)); - fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0))); + fileBlob.parameters().add(new StringValue(fileData.fileReference().value())); + fileBlob.parameters().add(new StringValue(fileData.filename())); + fileBlob.parameters().add(new StringValue(fileData.type().name())); + fileBlob.parameters().add(new DataValue(fileData.content())); + fileBlob.parameters().add(new Int64Value(fileData.xxhash())); fileBlob.parameters().add(new Int32Value(status.getCode())); fileBlob.parameters().add(new StringValue(status.getDescription())); target.invokeSync(fileBlob, 600); if (fileBlob.isError()) { - log.warning("Failed delivering reference '" + reference.value() + "' with file '" + filename + "' to " + - target.toString() + " with error : '" + fileBlob.errorMessage() + "'."); + log.warning("Failed delivering reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + + target.toString() + " with error: '" + fileBlob.errorMessage() + "'."); } } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java new file mode 100644 index 00000000000..ad807f9527f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.io.IOUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertTrue; + +public class FileDirectoryTest { + + private FileDirectory fileDirectory; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Before + public void setup() { + fileDirectory = new FileDirectory(temporaryFolder.getRoot()); + } + + @Test + public void requireThatFileReferenceWithFilesWorks() throws IOException { + FileReference foo = createFile("foo"); + FileReference bar = createFile("bar"); + + assertTrue(fileDirectory.getFile(foo).exists()); + assertTrue(fileDirectory.getFile(bar).exists()); + } + + + @Test + public void requireThatFileReferenceWithSubDirectoriesWorks() throws IOException { + FileDirectory fileDirectory = new FileDirectory(temporaryFolder.getRoot()); + + FileReference foo = createFileInSubDir("subdir", "foo"); + FileReference bar = createFileInSubDir("subdir", "bar"); + + assertTrue(fileDirectory.getFile(foo).exists()); + assertTrue(fileDirectory.getFile(bar).exists()); + } + + // Content in created file is equal to the filename string + private FileReference createFile(String filename) throws IOException { + File file = temporaryFolder.newFile(filename); + IOUtils.writeFile(file, filename, false); + return fileDirectory.addFile(file); + } + + private FileReference createFileInSubDir(String subdirName, String filename) throws IOException { + File subDirectory = new File(temporaryFolder.getRoot(), subdirName); + if (!subDirectory.exists()) + subDirectory.mkdirs(); + File file = new File(subDirectory, filename); + IOUtils.writeFile(file, filename, false); + return fileDirectory.addFile(file); + } + + +} + + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 09260987ac0..5fcaee6e590 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.FileReference; import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; +import com.yahoo.vespa.filedistribution.FileReferenceData; import org.junit.Test; import java.io.File; @@ -35,7 +35,7 @@ public class FileServerTest { } @Test - public void requireThatExistingFileCanbeFound() throws IOException { + public void requireThatExistingFileCanBeFound() throws IOException { createCleanDir("123"); IOUtils.writeFile("123/f1", "test", true); assertTrue(fs.hasFile("123")); @@ -50,15 +50,13 @@ public class FileServerTest { cleanup(); } - private static class FileReceiver implements FileServer.Receiver { - CompletableFuture<byte []> content; - FileReceiver(CompletableFuture<byte []> content) { - this.content = content; - } - @Override - public void receive(FileReference reference, String filename, byte[] content, FileServer.ReplayStatus status) { - this.content.complete(content); - } + @Test + public void requireThatFileReferenceWithDirectoryCanBeFound() throws IOException { + createCleanDir("124/subdir"); + IOUtils.writeFile("124/subdir/f1", "test", false); + IOUtils.writeFile("124/subdir/f2", "test", false); + assertTrue(fs.hasFile("124/subdir")); + cleanup(); } @Test @@ -98,6 +96,17 @@ public class FileServerTest { assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); } + private static class FileReceiver implements FileServer.Receiver { + CompletableFuture<byte []> content; + FileReceiver(CompletableFuture<byte []> content) { + this.content = content; + } + @Override + public void receive(FileReferenceData fileData, FileServer.ReplayStatus status) { + this.content.complete(fileData.content()); + } + } + private void cleanup() { created.forEach((file) -> IOUtils.recursiveDeleteDir(file)); created.clear(); diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java index a8be7c5ed13..85026296363 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -70,7 +70,7 @@ public class JSONFormatter { Principal sslPrincipal = accessLogEntry.getSslPrincipal(); if (sslPrincipal != null) { - generator.writeStringField("ssl-principal", principal.getName()); + generator.writeStringField("ssl-principal", sslPrincipal.getName()); } // Only add remote address/port fields if relevant diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 5479cace414..2e23e550352 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -6,7 +6,8 @@ <artifactId>container-dependency-versions</artifactId> <packaging>pom</packaging> <version>6-SNAPSHOT</version> - <description>Dependency versions for all Vespa maven projects.</description> + <name>container-dependency-versions</name> + <description>Versions for all 3rd party dependencies provided from the Vespa container.</description> <url>https://github.com/vespa-engine</url> <licenses> @@ -15,33 +16,51 @@ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> </license> </licenses> + <developers> + <developer> + <name>Vespa</name> + <url>https://github.com/vespa-engine</url> + </developer> + </developers> + <distributionManagement> + <repository> + <id>bintray-vespa-repo</id> + <url>https://api.bintray.com/maven/yahoo/maven/vespa;publish=1</url> + </repository> + </distributionManagement> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> -<!-- TODO: add pluginManagement for bundle-plugin and/or compiler-plugin? - <build> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.6.1</version> - <configuration> - <source>1.8</source> - <target>1.8</target> - <optimize>true</optimize> - </configuration> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${project.version}</version> - <configuration> - <configGenVersion>${project.version}</configGenVersion> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> ---> + + <!-- TODO: add pluginManagement for bundle-plugin and/or compiler-plugin? + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.6.1</version> + <configuration> + <source>1.8</source> + <target>1.8</target> + <optimize>true</optimize> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${project.version}</version> + <configuration> + <configGenVersion>${project.version}</configGenVersion> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + --> <dependencyManagement> <dependencies> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ApplicationAction.java index 8614414dc95..3323cda89b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ApplicationAction.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; /** * @author bjorncs diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java index b6a21f94f74..a2a16d10cdb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; /** * @author bjorncs diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzIdentity.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentity.java index 08d9b02f0e8..ef63ef2581f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzIdentity.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentity.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzPrincipal.java index 613f122b831..8279edcd8e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzPrincipal.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; @@ -12,9 +12,12 @@ import java.util.Objects; public class AthenzPrincipal implements Principal { private final AthenzIdentity athenzIdentity; + private final NToken nToken; - public AthenzPrincipal(AthenzIdentity athenzIdentity) { + public AthenzPrincipal(AthenzIdentity athenzIdentity, + NToken nToken) { this.athenzIdentity = athenzIdentity; + this.nToken = nToken; } public AthenzIdentity getIdentity() { @@ -30,6 +33,10 @@ public class AthenzPrincipal implements Principal { return athenzIdentity.getDomain(); } + public NToken getNToken() { + return nToken; + } + @Override public String toString() { return "AthenzPrincipal{" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzPublicKey.java index 01596ead0f4..c7f370dd4e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzPublicKey.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import java.security.PublicKey; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzService.java index fbbc45e856a..24cd7671d96 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzService.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUser.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzUser.java index 32ebc4d78b4..782876f21f1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUser.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzUser.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzUtils.java index 6517d97151b..0ed5d86dd7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzUtils.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java index e41bd8d4283..1df1746b02e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; /** * @author bjorncs diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/NToken.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/NToken.java new file mode 100644 index 00000000000..c2796befdc8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/NToken.java @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import java.util.Objects; + +/** + * Represents an Athenz NToken (principal token) + * + * @author bjorncs + */ +public class NToken { + + private final String rawToken; + + public NToken(String rawToken) { + this.rawToken = rawToken; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NToken nToken = (NToken) o; + return Objects.equals(rawToken, nToken.rawToken); + } + + public String getRawToken() { + return rawToken; + } + + @Override + public int hashCode() { + return Objects.hash(rawToken); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZToken.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZToken.java new file mode 100644 index 00000000000..cfa63b04197 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZToken.java @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import java.util.Objects; + +/** + * Represents an Athenz ZToken (role token) + * + * @author bjorncs + */ +public class ZToken { + + private final String rawToken; + + public ZToken(String rawToken) { + this.rawToken = rawToken; + } + + public String getRawToken() { + return rawToken; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZToken zToken = (ZToken) o; + return Objects.equals(rawToken, zToken.rawToken); + } + + @Override + public int hashCode() { + return Objects.hash(rawToken); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java index 09673721f9d..d72b8960427 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsException.java new file mode 100644 index 00000000000..31e9e549c08 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsException.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +/** + * @author bjorncs + */ +public class ZmsException extends RuntimeException { + + private final int code; + + public ZmsException(int code, Throwable cause) { + super(cause.getMessage(), cause); + this.code = code; + } + + public ZmsException(int code, String message) { + super(message); + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java index 93fed95c768..e2cb38a8466 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import java.security.PublicKey; import java.util.Optional; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClient.java index 5db3c55e9ce..c7a2adfb17e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClient.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsException.java index cb0b21ba459..2be998e1544 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsException.java @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.athenz.zts.ZTSClientException; +package com.yahoo.vespa.hosted.controller.api.integration.athenz; /** * @author bjorncs @@ -10,12 +8,11 @@ public class ZtsException extends RuntimeException { private final int code; - public ZtsException(ZTSClientException e) { - super(e.getMessage(), e); - this.code = e.getCode(); + public ZtsException(int code, Throwable cause) { + super(cause.getMessage(), cause); + this.code = code; } - public int getCode() { return code; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/package-info.java new file mode 100644 index 00000000000..d66525275bc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.athenz; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java index 97d99e262b5..1f1350ca001 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -18,7 +17,6 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -44,32 +42,28 @@ public interface ZoneApiV2 { Response proxyGet( @PathParam("environment") String env, @PathParam("region") String region, - @PathParam("proxy_request") String proxyRequest, - @Context HttpServletRequest request); + @PathParam("proxy_request") String proxyRequest); @POST @Path("/{environment}/{region}/{proxy_request: .+}") Response proxyPost( @PathParam("environment") String env, @PathParam("region") String region, - @PathParam("proxy_request") String proxyRequest, - @Context HttpServletRequest request); + @PathParam("proxy_request") String proxyRequest); @PUT @Path("/{environment}/{region}/{proxy_request: .+}") Response proxyPut( @PathParam("environment") String env, @PathParam("region") String region, - @PathParam("proxy_request") String proxyRequest, - @Context HttpServletRequest request); + @PathParam("proxy_request") String proxyRequest); @DELETE @Path("/{environment}/{region}/{proxy_request: .+}") Response proxyDelete( @PathParam("environment") String env, @PathParam("region") String region, - @PathParam("proxy_request") String proxyRequest, - @Context HttpServletRequest request); + @PathParam("proxy_request") String proxyRequest); // Explicit mappings of some proxy requests (to enable creation of proxy clients with javax.ws.rs) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 9767ae57bf0..520fb839b51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -39,10 +39,10 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.maintenance.DeploymentExpirer; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index c9f58b7222c..71a0a7f6297 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -24,7 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 1dfe92f64e1..a52098a4a0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -12,11 +12,11 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.PersistenceException; @@ -200,8 +200,7 @@ public class TenantController { try (Lock lock = lock(tenantId)) { Tenant existing = tenant(tenantId).orElseThrow(() -> new NotExistsException(tenantId)); if (existing.isAthensTenant()) return existing; // nothing to do - log.info("Starting migration of " + existing + " to Athenz domain " + tenantDomain.id() + - " using " + nToken.getPrincipal()); + log.info("Starting migration of " + existing + " to Athenz domain " + tenantDomain.id()); if (tenantHaving(tenantDomain).isPresent()) throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " + "This domain is already used by " + tenantHaving(tenantDomain).get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java deleted file mode 100644 index 24ebe337e1f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.athenz.auth.token.PrincipalToken; -import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; - -import java.security.PrivateKey; -import java.security.PublicKey; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.concurrent.TimeUnit; - -/** - * Represents an Athenz NToken (principal token) - * - * @author bjorncs - */ -// TODO Split out encoding/decoding of token into separate class. Move NToken to controller-api. -public class NToken { - - // Max allowed skew in token timestamp (only for creation, not expiry timestamp) - private static final int ALLOWED_TIMESTAMP_OFFSET = (int) TimeUnit.SECONDS.toSeconds(300); - - private final PrincipalToken token; - - // Note: PrincipalToken does not provide any way of constructing an instance from a unsigned token string - public NToken(String signedToken) { - try { - this.token = new PrincipalToken(signedToken); - if (this.token.getSignature() == null) { - throw new IllegalArgumentException("Signature missing (unsigned token)"); - } - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Malformed NToken: " + e.getMessage()); - } - } - - public AthenzPrincipal getPrincipal() { - return new AthenzPrincipal(getAthenzIdentity()); - } - - public AthenzIdentity getAthenzIdentity() { - return AthenzUtils.createAthenzIdentity(getDomain(), token.getName()); - } - - public AthenzDomain getDomain() { - return new AthenzDomain(token.getDomain()); - } - - public String getToken() { - return token.getSignedToken(); - } - - public String getKeyId() { - return token.getKeyId(); - } - - public void validateSignatureAndExpiration(PublicKey publicKey) throws InvalidTokenException { - StringBuilder errorMessageBuilder = new StringBuilder(); - if (!token.validate(publicKey, ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) { - throw new InvalidTokenException("NToken is expired or has invalid signature: " + errorMessageBuilder.toString()); - } - } - - @Override - public String toString() { - return String.format("NToken(%s)", token.getUnsignedToken()); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NToken nToken = (NToken) o; - return Objects.equals(getToken(), nToken.getToken()); // PrincipalToken does not implement equals() - } - - @Override - public int hashCode() { - return Objects.hash(getToken()); // PrincipalToken does not implement hashcode() - } - - public static class Builder { - - private final String version; - private final AthenzIdentity identity; - private final PrivateKey privateKey; - private final String keyId; - private Optional<String> salt = Optional.empty(); - private Optional<String> hostname = Optional.empty(); - private Optional<String> ip = Optional.empty(); - private OptionalLong issueTime = OptionalLong.empty(); - private OptionalLong expirationWindow = OptionalLong.empty(); - - /** - * NOTE: We must have some signature, else we might end up with problems later on as - * {@link PrincipalToken#PrincipalToken(String)} only accepts signed token - * (supplying an unsigned token to the constructor will result in inconsistent state) - */ - public Builder(String version, AthenzIdentity identity, PrivateKey privateKey, String keyId) { - this.version = version; - this.identity = identity; - this.privateKey = privateKey; - this.keyId = keyId; - } - - public Builder salt(String salt) { - this.salt = Optional.of(salt); - return this; - } - - public Builder hostname(String hostname) { - this.hostname = Optional.of(hostname); - return this; - } - - public Builder ip(String ip) { - this.ip = Optional.of(ip); - return this; - } - - public Builder issueTime(long issueTime) { - this.issueTime = OptionalLong.of(issueTime); - return this; - } - - public Builder expirationWindow(long expirationWindow) { - this.expirationWindow = OptionalLong.of(expirationWindow); - return this; - } - - public NToken build() { - PrincipalToken token = new PrincipalToken.Builder(version, identity.getDomain().id(), identity.getName()) - .keyId(this.keyId) - .salt(this.salt.orElse(null)) - .host(this.hostname.orElse(null)) - .ip(this.ip.orElse(null)) - .issueTime(this.issueTime.orElse(0)) - .expirationWindow(this.expirationWindow.orElse(0)) - .build(); - token.sign(this.privateKey); - return new NToken(token.getSignedToken()); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java deleted file mode 100644 index 59548339d11..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.athenz.zms.ZMSClientException; - -/** - * @author bjorncs - */ -public class ZmsException extends RuntimeException { - - private final int code; - - public ZmsException(ZMSClientException e) { - super(e.getMessage(), e); - this.code = e.getCode(); - } - - - public int getCode() { - return code; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java index 51865be04fa..328461355db 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java @@ -6,10 +6,10 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.util.concurrent.Executor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java index f43d2d8e80e..69f59ebabe2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java @@ -1,17 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.filter; +import com.yahoo.athenz.auth.token.PrincipalToken; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import java.security.PublicKey; +import java.time.Duration; import java.util.Optional; import java.util.logging.Logger; -import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; +import static com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; /** * Validates the content of an NToken: @@ -22,6 +26,9 @@ import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.ZMS_ATHENZ_SE */ class NTokenValidator { + // Max allowed skew in token timestamp (only for creation, not expiry timestamp) + private static final long ALLOWED_TIMESTAMP_OFFSET = Duration.ofMinutes(5).getSeconds(); + private static final Logger log = Logger.getLogger(NTokenValidator.class.getName()); private final ZmsKeystore keystore; @@ -35,10 +42,15 @@ class NTokenValidator { } AthenzPrincipal validate(NToken token) throws InvalidTokenException { - PublicKey zmsPublicKey = getPublicKey(token.getKeyId()) + PrincipalToken principalToken = new PrincipalToken(token.getRawToken()); + PublicKey zmsPublicKey = getPublicKey(principalToken.getKeyId()) .orElseThrow(() -> new InvalidTokenException("NToken has an unknown keyId")); - validateSignatureAndExpiration(token, zmsPublicKey); - return token.getPrincipal(); + validateSignatureAndExpiration(principalToken, zmsPublicKey); + return new AthenzPrincipal( + AthenzUtils.createAthenzIdentity( + new AthenzDomain(principalToken.getDomain()), + principalToken.getName()), + token); } private Optional<PublicKey> getPublicKey(String keyId) throws InvalidTokenException { @@ -50,13 +62,13 @@ class NTokenValidator { } } - private static void validateSignatureAndExpiration(NToken token, PublicKey zmsPublicKey) throws InvalidTokenException { - try { - token.validateSignatureAndExpiration(zmsPublicKey); - } catch (InvalidTokenException e) { - // The underlying error message is not user friendly - logDebug(e.getMessage()); - throw new InvalidTokenException("NToken is expired or has invalid signature"); + private static void validateSignatureAndExpiration(PrincipalToken token, + PublicKey zmsPublicKey) throws InvalidTokenException { + StringBuilder errorMessageBuilder = new StringBuilder(); + if (!token.validate(zmsPublicKey, (int) ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) { + String message = "NToken is expired or has invalid signature: " + errorMessageBuilder.toString(); + logDebug(message); + throw new InvalidTokenException(message); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java index d4a5b691350..b4859220667 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java @@ -8,13 +8,15 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import com.yahoo.vespa.hosted.controller.restapi.application.Authorizer; import java.security.Principal; +import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Logger; import java.util.stream.Stream; @@ -34,11 +36,13 @@ public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter { private static final Logger log = Logger.getLogger(UserAuthWithAthenzPrincipalFilter.class.getName()); private final String userAuthenticationPassThruAttribute; + private final String principalHeaderName; @Inject public UserAuthWithAthenzPrincipalFilter(ZmsKeystore zmsKeystore, Executor executor, AthenzConfig config) { super(zmsKeystore, executor, config); this.userAuthenticationPassThruAttribute = config.userAuthenticationPassThruAttribute(); + this.principalHeaderName = config.principalHeaderName(); } @Override @@ -81,13 +85,14 @@ public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter { * NOTE: The Bouncer user roles ({@link DiscFilterRequest#roles} are still intact as they are required * for {@link Authorizer#isMemberOfVespaBouncerGroup(HttpRequest)}. */ - private static void rewriteUserPrincipalToAthenz(DiscFilterRequest request) { + private void rewriteUserPrincipalToAthenz(DiscFilterRequest request) { Principal userPrincipal = request.getUserPrincipal(); log.log(LogLevel.DEBUG, () -> "Original user principal: " + userPrincipal.toString()); UserId userId = new UserId(userPrincipal.getName()); AthenzUser athenzIdentity = AthenzUser.fromUserId(userId); request.setRemoteUser(athenzIdentity.getFullName()); - request.setUserPrincipal(new AthenzPrincipal(athenzIdentity)); + NToken nToken = Optional.ofNullable(request.getHeader(principalHeaderName)).map(NToken::new).orElse(null); + request.setUserPrincipal(new AthenzPrincipal(athenzIdentity, nToken)); } private enum UserAuthenticationResult { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 1c32b35f599..540cdb39630 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -10,17 +10,17 @@ import com.yahoo.athenz.auth.token.PrincipalToken; import com.yahoo.athenz.auth.util.Crypto; import com.yahoo.athenz.zms.ZMSClient; import com.yahoo.athenz.zts.ZTSClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; import com.yahoo.vespa.hosted.controller.api.integration.security.KeyService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.security.PrivateKey; import java.util.concurrent.TimeUnit; -import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.USER_PRINCIPAL_DOMAIN; +import static com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils.USER_PRINCIPAL_DOMAIN; /** * @author bjorncs @@ -59,7 +59,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { */ @Override public ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken) { - PrincipalToken signedToken = new PrincipalToken(authorizedServiceToken.getToken()); + PrincipalToken signedToken = new PrincipalToken(authorizedServiceToken.getRawToken()); AthenzConfig.Service service = config.service(); signedToken.signForAuthorizedService( config.domain() + "." + service.name(), service.publicKeyId(), getServicePrivateKey()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java index beca72cc3ca..d3fac257583 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java @@ -13,12 +13,12 @@ import com.yahoo.athenz.zms.ZMSClientException; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.util.Arrays; @@ -179,7 +179,7 @@ public class ZmsClientImpl implements ZmsClient { wrappedCode.run(); } catch (ZMSClientException e) { logWarning(e); - throw new ZmsException(e); + throw new ZmsException(e.getCode(), e); } } @@ -188,7 +188,7 @@ public class ZmsClientImpl implements ZmsClient { return wrappedCode.get(); } catch (ZMSClientException e) { logWarning(e); - throw new ZmsException(e); + throw new ZmsException(e.getCode(), e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java index fd58a3daba7..513434f7273 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java @@ -3,11 +3,11 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import java.security.PublicKey; import java.util.List; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java index b141d728afd..76c000936e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java @@ -6,10 +6,10 @@ import com.yahoo.athenz.zts.ZTSClient; import com.yahoo.athenz.zts.ZTSClientException; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZtsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsException; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.util.List; @@ -44,7 +44,7 @@ public class ZtsClientImpl implements ZtsClient { .map(AthenzDomain::new) .collect(toList()); } catch (ZTSClientException e) { - throw new ZtsException(e); + throw new ZtsException(e.getCode(), e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java index d4a2d77c115..52a1f2d477d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; import java.util.logging.Level; import java.util.logging.Logger; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java index b6dc8aa7e25..c633d780e30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; import java.util.HashMap; import java.util.HashSet; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index 1f37edf5526..4b50a34094a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -1,15 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.mock; -import com.yahoo.athenz.zms.ZMSClientException; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import java.util.ArrayList; import java.util.List; @@ -111,7 +110,7 @@ public class ZmsClientMock implements ZmsClient { } private static ZmsException zmsException(int code, String message, Object... args) { - return new ZmsException(new ZMSClientException(code, String.format(message, args))); + return new ZmsException(code, String.format(message, args)); } private static void log(String format, Object... args) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java index ee983d851ce..ee43181a358 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; import java.util.List; import java.util.logging.Level; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index f7810744b71..d64f07906ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -64,12 +64,12 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java index 9e1ccbdcac1..b7080a763f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java @@ -10,11 +10,11 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; -import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; import com.yahoo.vespa.hosted.controller.common.ContextAttributes; import com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java index 8b548d96f62..c7e03048ec8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java @@ -7,11 +7,11 @@ import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import javax.ws.rs.ForbiddenException; import javax.ws.rs.NotAuthorizedException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index d152cf80472..0e07f7b7589 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; @@ -119,15 +120,16 @@ public class VersionStatus { private static ListMap<Version, String> findConfigServerVersions(Controller controller) { List<URI> configServers = controller.zoneRegistry().zones().stream() - .flatMap(zone -> controller.getConfigServerUris(zone.environment(), zone.region()).stream()) - .collect(Collectors.toList()); + .filter(zone -> ! zone.region().equals(RegionName.from("us-east-2a"))) + .flatMap(zone -> controller.getConfigServerUris(zone.environment(), zone.region()).stream()) + .collect(Collectors.toList()); ListMap<Version, String> versions = new ListMap<>(); for (URI configServer : configServers) versions.put(controller.applications().configserverClient().version(configServer), configServer.getHost()); return versions; } - + private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions, List<Application> applications, Instant jobTimeoutLimit) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 9d4652941da..66829a4f77e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -30,7 +30,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java index 9dbebecdb25..085819b433d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java @@ -9,9 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.filter.AthenzTestUtils; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; /** * @author Tony Vaagenes @@ -34,8 +32,6 @@ public class TestIdentities { public static Tenant tenant = Tenant.createOpsDbTenant(tenantId, userGroup1, property); - public static NToken userNToken = new NToken.Builder( - "U1", AthenzUser.fromUserId(userId), AthenzTestUtils.generateRsaKeypair().getPrivate(), "0") - .build(); + public static NToken userNToken = new NToken("dummy"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java index 5a3f048101a..ffb78b7342a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java @@ -7,10 +7,10 @@ import com.yahoo.jdisc.handler.ReadableContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; -import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; import org.junit.Before; import org.junit.Test; @@ -35,7 +35,7 @@ import static org.mockito.Mockito.when; */ public class AthenzPrincipalFilterTest { - private static final NToken NTOKEN = createDummyToken(); + private static final NToken NTOKEN = new NToken("dummy"); private static final String ATHENZ_PRINCIPAL_HEADER = "Athenz-Principal-Auth"; private NTokenValidator validator; @@ -44,13 +44,13 @@ public class AthenzPrincipalFilterTest { @Before public void before() { validator = mock(NTokenValidator.class); - principal = new AthenzPrincipal(AthenzUser.fromUserId(new UserId("bob"))); + principal = new AthenzPrincipal(AthenzUser.fromUserId(new UserId("bob")), NTOKEN); } @Test public void valid_ntoken_is_accepted() throws Exception { DiscFilterRequest request = mock(DiscFilterRequest.class); - when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getToken()); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken()); when(validator.validate(NTOKEN)).thenReturn(principal); @@ -78,7 +78,7 @@ public class AthenzPrincipalFilterTest { @Test public void invalid_token_is_unauthorized() throws Exception { DiscFilterRequest request = mock(DiscFilterRequest.class); - when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getToken()); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken()); when(validator.validate(NTOKEN)).thenThrow(new InvalidTokenException("Invalid token")); @@ -92,12 +92,6 @@ public class AthenzPrincipalFilterTest { assertThat(responseHandler.getResponseContent(), containsString("Invalid token")); } - private static NToken createDummyToken() { - return new NToken.Builder( - "U1", AthenzUser.fromUserId(new UserId("bob")), AthenzTestUtils.generateRsaKeypair().getPrivate(), "0") - .build(); - } - private static class ResponseHandlerMock implements ResponseHandler { public Response response; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java index 927910a89a1..907fabe9d75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java @@ -1,22 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.filter; +import com.yahoo.athenz.auth.token.PrincipalToken; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; -import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.security.KeyPair; import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.time.Instant; import java.util.Optional; -import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; +import static com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; import static org.junit.Assert.assertEquals; /** @@ -34,7 +37,7 @@ public class NTokenValidatorTest { @Test public void valid_token_is_accepted() throws NoSuchAlgorithmException, InvalidTokenException { NTokenValidator validator = new NTokenValidator(createKeystore()); - NToken token = createNToken(IDENTITY, System.currentTimeMillis(), TRUSTED_KEY, "0"); + NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "0"); AthenzPrincipal principal = validator.validate(token); assertEquals("user.myuser", principal.getIdentity().getFullName()); } @@ -42,7 +45,7 @@ public class NTokenValidatorTest { @Test public void invalid_signature_is_not_accepted() throws InvalidTokenException { NTokenValidator validator = new NTokenValidator(createKeystore()); - NToken token = createNToken(IDENTITY, System.currentTimeMillis(), UNKNOWN_KEY, "0"); + NToken token = createNToken(IDENTITY, Instant.now(), UNKNOWN_KEY.getPrivate(), "0"); exceptionRule.expect(InvalidTokenException.class); exceptionRule.expectMessage("NToken is expired or has invalid signature"); validator.validate(token); @@ -51,7 +54,7 @@ public class NTokenValidatorTest { @Test public void expired_token_is_not_accepted() throws InvalidTokenException { NTokenValidator validator = new NTokenValidator(createKeystore()); - NToken token = createNToken(IDENTITY, 1234 /*long time ago*/, TRUSTED_KEY, "0"); + NToken token = createNToken(IDENTITY, Instant.ofEpochMilli(1234) /*long time ago*/, TRUSTED_KEY.getPrivate(), "0"); exceptionRule.expect(InvalidTokenException.class); exceptionRule.expectMessage("NToken is expired or has invalid signature"); validator.validate(token); @@ -60,7 +63,7 @@ public class NTokenValidatorTest { @Test public void unknown_keyId_is_not_accepted() throws InvalidTokenException { NTokenValidator validator = new NTokenValidator(createKeystore()); - NToken token = createNToken(IDENTITY, System.currentTimeMillis(), TRUSTED_KEY, "unknown-key-id"); + NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "unknown-key-id"); exceptionRule.expect(InvalidTokenException.class); exceptionRule.expectMessage("NToken has an unknown keyId"); validator.validate(token); @@ -70,7 +73,7 @@ public class NTokenValidatorTest { public void failing_to_find_key_should_throw_exception() throws InvalidTokenException { ZmsKeystore keystore = (athensService, keyId) -> { throw new RuntimeException(); }; NTokenValidator validator = new NTokenValidator(keystore); - NToken token = createNToken(IDENTITY, System.currentTimeMillis(), TRUSTED_KEY, "0"); + NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "0"); exceptionRule.expect(InvalidTokenException.class); exceptionRule.expectMessage("Failed to retrieve public key"); validator.validate(token); @@ -83,14 +86,17 @@ public class NTokenValidatorTest { : Optional.empty(); } - private static NToken createNToken(AthenzIdentity identity, long issueTime, KeyPair keyPair, String keyId) { - return new NToken.Builder("U1", identity, keyPair.getPrivate(), keyId) + private static NToken createNToken(AthenzIdentity identity, Instant issueTime, PrivateKey privateKey, String keyId) { + PrincipalToken token = new PrincipalToken.Builder("U1", identity.getDomain().id(), identity.getName()) + .keyId(keyId) .salt("1234") - .hostname("host") + .host("host") .ip("1.2.3.4") - .issueTime(issueTime / 1000) + .issueTime(issueTime.getEpochSecond()) .expirationWindow(1000) .build(); + token.sign(privateKey); + return new NToken(token.getSignedToken()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 9f5de1e460b..bb5f129b20b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -22,9 +22,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d12f78a1db3..f48f6b02bd2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -22,10 +22,10 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java index 5899c767beb..988304be600 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java @@ -6,10 +6,10 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TestIdentities; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; -import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; import javax.ws.rs.core.SecurityContext; import java.security.Principal; @@ -37,7 +37,8 @@ public class MockAuthorizer extends Authorizer { if (domain == null || name == null) return Optional.empty(); return Optional.of( new AthenzPrincipal( - AthenzUtils.createAthenzIdentity(new AthenzDomain(domain), name))); + AthenzUtils.createAthenzIdentity(new AthenzDomain(domain), name), + new NToken("dummy"))); } /** Returns the hardcoded NToken of {@link TestIdentities#userId} */ diff --git a/dist/vespa.spec b/dist/vespa.spec index dcfe5719650..c17454804dc 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -68,6 +68,8 @@ BuildRequires: make BuildRequires: vespa-cppunit-devel >= 1.12.1-6 BuildRequires: vespa-libtorrent-devel >= 1.0.11-6 BuildRequires: systemd +BuildRequires: flex >= 2.5.0 +BuildRequires: bison >= 3.0.0 %if 0%{?centos} Requires: epel-release %endif diff --git a/document/src/main/java/com/yahoo/document/idstring/IdIdString.java b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java index 7fcb530b347..44ff08c73f0 100644 --- a/document/src/main/java/com/yahoo/document/idstring/IdIdString.java +++ b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java @@ -62,6 +62,9 @@ public class IdIdString extends IdString { if (hasSetLocation) { throw new IllegalArgumentException("Illegal key combination in " + keyValues); } + if (value.isEmpty()) { + throw new IllegalArgumentException("ID location value for 'n=' key is empty"); + } location = Long.parseLong(value); hasSetLocation = true; hasNumber = true; @@ -70,6 +73,9 @@ public class IdIdString extends IdString { if (hasSetLocation) { throw new IllegalArgumentException("Illegal key combination in " + keyValues); } + if (value.isEmpty()) { + throw new IllegalArgumentException("ID location value for 'g=' key is empty"); + } location = makeLocation(value); hasSetLocation = true; hasGroup = true; diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java index 79a10bc72e4..bd769889363 100644 --- a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.document.*; import com.yahoo.document.idstring.*; import com.yahoo.vespa.objects.BufferSerializer; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; import java.math.BigInteger; @@ -12,16 +15,20 @@ import java.util.regex.Pattern; import java.util.Arrays; import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -public class DocumentIdTestCase extends junit.framework.TestCase { +public class DocumentIdTestCase { DocumentTypeManager manager = new DocumentTypeManager(); - public DocumentIdTestCase(String name) { - super(name); - } + @Rule + public ExpectedException expectedException = ExpectedException.none(); - protected void setUp() { + @Before + public void setUp() { DocumentType testDocType = new DocumentType("testdoc"); testDocType.addHeaderField("intattr", DataType.INT); @@ -33,6 +40,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { manager.registerDocumentType(testDocType); } + @Test public void testCompareTo() { DocumentId docId1 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.uio.no/")).getId(); DocumentId docId2 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.uio.no/")).getId(); @@ -56,6 +64,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { } } + @Test public void testValidInvalidUriSchemes() { try { //valid URIs @@ -92,9 +101,23 @@ public class DocumentIdTestCase extends junit.framework.TestCase { checkInvalidUri("id:namespace:type:n=0,g=foo:foo"); } + @Test + public void empty_user_location_value_throws_exception() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("ID location value for 'n=' key is empty"); + new DocumentId("id:namespace:type:n=:foo"); + } + + @Test + public void empty_group_location_value_throws_exception() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("ID location value for 'g=' key is empty"); + new DocumentId("id:namespace:type:g=:foo"); + } //Compares globalId with C++ implementation located in // ~document-HEAD/document/src/tests/cpp-globalidbucketids.txt + @Test public void testCalculateGlobalId() throws IOException{ String file = "src/tests/cpp-globalidbucketids.txt"; @@ -135,6 +158,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { //Compares bucketId with C++ implementation located in // ~document-HEAD/document/src/tests/cpp-globalidbucketids.txt + @Test public void testGetBucketId() throws IOException{ String file = "src/tests/cpp-globalidbucketids.txt"; BufferedReader fr = new BufferedReader(new FileReader(file)); @@ -153,6 +177,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { fr.close(); } + @Test public void testGroupdoc() { try { //valid @@ -166,11 +191,13 @@ public class DocumentIdTestCase extends junit.framework.TestCase { } } + @Test public void testInvalidGroupdoc() { checkInvalidUri("grouppdoc:blabla:something"); checkInvalidUri("groupdoc:blablasomething"); } + @Test public void testUriNamespace() { DocumentId docId = new DocumentId("doc:bar:foo"); assertEquals("doc:bar:foo", docId.toString()); @@ -217,6 +244,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { assertEquals(1268182861, ((OrderDocIdString)docId.getScheme()).getOrdering()); } + @Test public void testIdStrings() { DocumentId docId; docId = new DocumentId(new DocIdString("test", "baaaa")); @@ -240,6 +268,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { assertEquals("type", docId.getDocType()); } + @Test public void testIdStringFeatures() { DocumentId none = new DocumentId("id:ns:type::foo"); assertFalse(none.getScheme().hasGroup()); @@ -276,6 +305,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { assertEquals(42, order.getScheme().getNumber()); } + @Test public void testHashCodeOfGids() { DocumentId docId0 = new DocumentId("doc:blabla:0"); byte[] docId0Gid = docId0.getGlobalId(); @@ -295,6 +325,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { assertEquals(Arrays.hashCode(docId0Gid), Arrays.hashCode(docId0CopyGid)); } + @Test public void testDocumentIdCanOnlyContainTextCharacters() throws UnsupportedEncodingException { assertExceptionWhenConstructing(new byte[]{105, 100, 58, 97, 58, 98, 58, 58, 0, 99}, // "id:a:b::0x0c" "illegal code point 0x0"); @@ -313,6 +344,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { } } + @Test public void testSerializedDocumentIdCanContainNonTextCharacter() throws UnsupportedEncodingException { String strId = new String(new byte[]{105, 100, 58, 97, 58, 98, 58, 58, 7, 99}); // "id:a:b::0x7c" DocumentId docId = DocumentId.createFromSerialized(strId); @@ -328,6 +360,7 @@ public class DocumentIdTestCase extends junit.framework.TestCase { } } + @Test public void testSerializedDocumentIdCannotContainZeroByte() throws UnsupportedEncodingException { String strId = new String(new byte[]{105, 100, 58, 97, 58, 98, 58, 58, 0, 99}); // "id:a:b::0x0c" try { diff --git a/document/src/tests/bucketselectortest.cpp b/document/src/tests/bucketselectortest.cpp index 0f8520745f1..e0857a32dba 100644 --- a/document/src/tests/bucketselectortest.cpp +++ b/document/src/tests/bucketselectortest.cpp @@ -85,8 +85,6 @@ void BucketSelectorTest::testSimple() ASSERT_BUCKET_COUNT("id.bucket == 0x4000000000000258", 1u); // Bucket 600 ASSERT_BUCKET_COUNT("(testdoctype1 and id.bucket=0)", 1u); - ASSERT_BUCKET_COUNT("searchcolumn.3 = 1", 21845u); - // Check that the correct buckets is found ASSERT_BUCKET("id = \"userdoc:ns:123:foobar\"", document::BucketId(58, 123)); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index c5715ae5114..db7b48cdc3a 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include <cppunit/TestFixture.h> #include <cppunit/extensions/HelperMacros.h> #include <vespa/document/repo/configbuilder.h> @@ -12,12 +11,16 @@ #include <vespa/document/select/visitor.h> #include <vespa/document/select/bodyfielddetector.h> #include <vespa/document/select/valuenode.h> +#include <vespa/document/select/valuenodes.h> #include <vespa/document/select/branch.h> #include <vespa/document/select/simpleparser.h> #include <vespa/document/select/constant.h> #include <vespa/document/select/invalidconstant.h> #include <vespa/document/select/doctype.h> #include <vespa/document/select/compare.h> +#include <vespa/document/select/parse_utils.h> +#include <vespa/vespalib/util/exceptions.h> +#include <limits> using namespace document::config_builder; @@ -34,6 +37,15 @@ class DocumentSelectParserTest : public CppUnit::TestFixture { CPPUNIT_TEST(testThatComplexFieldValuesHaveCorrectFieldNames); CPPUNIT_TEST(testBodyFieldDetection); CPPUNIT_TEST(testDocumentUpdates); + CPPUNIT_TEST(test_syntax_error_reporting); + CPPUNIT_TEST(test_operator_precedence); + CPPUNIT_TEST(test_token_used_as_ident_preserves_casing); + CPPUNIT_TEST(test_ambiguous_field_spec_expression_is_handled_correctly); + CPPUNIT_TEST(test_can_build_field_value_from_field_expr_node); + CPPUNIT_TEST(test_can_build_function_call_from_field_expr_node); + CPPUNIT_TEST(test_function_call_on_doctype_throws_exception); + CPPUNIT_TEST(test_parse_utilities_handle_well_formed_input); + CPPUNIT_TEST(test_parse_utilities_handle_malformed_input); CPPUNIT_TEST_SUITE_END(); BucketIdFactory _bucketIdFactory; @@ -51,11 +63,13 @@ class DocumentSelectParserTest : public CppUnit::TestFixture { const std::string& hstr); std::unique_ptr<select::FieldValueNode> - parseFieldValue(const std::string expression); + parseFieldValue(const std::string& expression); template <typename ContainsType> select::ResultList doParse(const vespalib::stringref& expr, const ContainsType& t); + + std::string parse_to_tree(const std::string& str); public: DocumentSelectParserTest() @@ -88,7 +102,15 @@ public: void testDocumentUpdates2(); void testDocumentUpdates3(); void testDocumentUpdates4(); - void testDocumentUpdates5(); + void test_syntax_error_reporting(); + void test_operator_precedence(); + void test_token_used_as_ident_preserves_casing(); + void test_ambiguous_field_spec_expression_is_handled_correctly(); + void test_can_build_field_value_from_field_expr_node(); + void test_can_build_function_call_from_field_expr_node(); + void test_function_call_on_doctype_throws_exception(); + void test_parse_utilities_handle_well_formed_input(); + void test_parse_utilities_handle_malformed_input(); }; CPPUNIT_TEST_SUITE_REGISTRATION(DocumentSelectParserTest); @@ -111,9 +133,9 @@ void DocumentSelectParserTest::setUp() builder.document(-1673092522, "usergroup", Struct("usergroup.header"), Struct("usergroup.body")); - _repo.reset(new DocumentTypeRepo(builder.config())); + _repo = std::make_unique<DocumentTypeRepo>(builder.config()); - _parser.reset(new select::Parser(*_repo, _bucketIdFactory)); + _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory); } Document::SP DocumentSelectParserTest::createDoc( @@ -319,11 +341,45 @@ void verifyParse(const std::string& query, const char* expected = 0) { } } +void DocumentSelectParserTest::test_syntax_error_reporting() { + createDocs(); + + verifyFailedParse("testdoctype1.headerval == aaa", "ParsingFailedException: " + "syntax error, unexpected end of input, expecting . at column 30 " + "when parsing selection 'testdoctype1.headerval == aaa'"); + // TODO improve error reporting of broken escape sequences. Current error messages + // are not too helpful since we simply fail to parse the string token altogether. + verifyFailedParse("testdoctype1.headerval == \"tab\\x0notcomplete\"", + "ParsingFailedException: Unexpected character: '\\\"' at column 27 " + "when parsing selection 'testdoctype1.headerval == \"tab\\x0notcomplete\"'"); + verifyFailedParse("testdoctype1.headerval == \"tab\\ysf\"", + "ParsingFailedException: Unexpected character: '\\\"' at column 27 " + "when parsing selection 'testdoctype1.headerval == \"tab\\ysf\"'"); + // Test illegal operator + verifyFailedParse("testdoctype1.headerval <> 12", "ParsingFailedException: syntax error, " + "unexpected > at column 25 when parsing selection 'testdoctype1.headerval <> 12'"); + + // This will trigger a missing doctype error instead of syntax error, as "fal" + // will be reduced into a doctype rule. + verifyFailedParse("fal se", "ParsingFailedException: Document type 'fal' " + "not found at column 1 when parsing selection 'fal se'"); + + verifyFailedParse("mytype", "ParsingFailedException: Document type 'mytype' not found"); + + verifyFailedParse("mytype.foo.bar", "ParsingFailedException: Document type 'mytype' not found"); + + verifyFailedParse("testdoctype1 == 8", "ParsingFailedException: syntax error, unexpected ==, " + "expecting end of input at column 14 when parsing selection 'testdoctype1 == 8'"); + + verifyFailedParse("(1 + 2)", "ParsingFailedException: expected field spec, " + "doctype, bool or comparison at column 1 when parsing selection '(1 + 2)'"); +} + void DocumentSelectParserTest::testParseTerminals() { createDocs(); - // Test number value + // Test number value verifyParse("", "true"); verifyParse("testdoctype1.headerval == 123"); verifyParse("testdoctype1.headerval == +123.53", "testdoctype1.headerval == 123.53"); @@ -332,10 +388,8 @@ void DocumentSelectParserTest::testParseTerminals() "testdoctype1.headerval == 2.34124e+08"); verifyParse("testdoctype1.headerval == -234123.523E-3", "testdoctype1.headerval == -234.124"); - verifyFailedParse("testdoctype1.headerval == aaa", "ParsingFailedException: " - "Unexpected token at position 23 ('== aaa') in query " - "'testdoctype1.headerval == aaa', at fullParse in "); - // Test string value + + // Test string value verifyParse("testdoctype1.headerval == \"test\""); std::unique_ptr<select::Node> node( _parser->parse("testdoctype1.headerval == \"test\"")); @@ -345,64 +399,46 @@ void DocumentSelectParserTest::testParseTerminals() dynamic_cast<const select::FieldValueNode&>(compnode.getLeft())); const select::StringValueNode& vnode( dynamic_cast<const select::StringValueNode&>(compnode.getRight())); - /* - CPPUNIT_ASSERT_EQUAL(vespalib::string("testdoctype1"), - fnode.getDocType()->getName()); - */ + CPPUNIT_ASSERT_EQUAL(vespalib::string("headerval"), fnode.getFieldName()); CPPUNIT_ASSERT_EQUAL(vespalib::string("test"), vnode.getValue()); - // Test whitespace + // Test whitespace verifyParse("testdoctype1.headerval == \"te st \""); verifyParse(" \t testdoctype1.headerval\t== \t \"test\"\t", "testdoctype1.headerval == \"test\""); - // Test escaping + // Test escaping verifyParse("testdoctype1.headerval == \"tab\\ttest\""); verifyParse("testdoctype1.headerval == \"tab\\x09test\"", "testdoctype1.headerval == \"tab\\ttest\""); verifyParse("testdoctype1.headerval == \"tab\\x055test\""); - verifyFailedParse("testdoctype1.headerval == \"tab\\x0notcomplete\"", - "ParsingFailedException: Unexpected token at position 23 " - "('== \"tab\\x0') in query 'testdoctype1.headerval == \"tab\\x0notcomplete\"', " - "at fullParse in "); - verifyFailedParse("testdoctype1.headerval == \"tab\\ysf\"", - "ParsingFailedException: Unexpected token at position 23 " - "('== \"tab\\ys') in query 'testdoctype1.headerval == \"tab\\ysf\"', " - "at fullParse in "); node = _parser->parse("testdoctype1.headerval == \"\\tt\\x48 \\n\""); select::Compare& escapednode(dynamic_cast<select::Compare&>(*node)); const select::StringValueNode& escval( dynamic_cast<const select::StringValueNode&>(escapednode.getRight())); CPPUNIT_ASSERT_EQUAL(vespalib::string("\ttH \n"), escval.getValue()); - // Test illegal operator - verifyFailedParse("testdoctype1.headerval <> 12", "ParsingFailedException: Unexpected" - " token at position 23 ('<> 12') in query 'testdoctype1.headerval <> 12', at"); - // Test <= <, > >= + // Test <= <, > >= verifyParse("testdoctype1.headerval >= 123"); verifyParse("testdoctype1.headerval > 123"); verifyParse("testdoctype1.headerval <= 123"); verifyParse("testdoctype1.headerval < 123"); verifyParse("testdoctype1.headerval != 123"); - // Test defined + // Test defined verifyParse("testdoctype1.headerval", "testdoctype1.headerval != null"); - // Test bools - verifyParse("TRUE"); - verifyParse("FALSE"); + // Test bools + verifyParse("TRUE", "true"); + verifyParse("FALSE", "false"); verifyParse("true"); verifyParse("false"); - verifyParse("faLSe"); - verifyFailedParse("fal se", "ParsingFailedException: Unexpected token at " - "position 4 ('se') in query 'fal se', at"); + verifyParse("faLSe", "false"); - // Test document types + // Test document types verifyParse("testdoctype1"); - verifyFailedParse("mytype", "ParsingFailedException: Document type mytype " - "not found"); verifyParse("_test_doctype3_"); verifyParse("_test_doctype3_._only_in_child_ == 0"); - // Test document id with simple parser. + // Test document id with simple parser. verifySimpleParse("id == \"userdoc:ns:mytest\""); verifySimpleParse("id.namespace == \"myspace\""); verifySimpleParse("id.scheme == \"userdoc\""); @@ -411,7 +447,7 @@ void DocumentSelectParserTest::testParseTerminals() verifySimpleParse("id.user == 1234"); verifySimpleParse("id.user == 0x12456ab", "id.user == 19158699"); - // Test document id + // Test document id verifyParse("id == \"userdoc:ns:mytest\""); verifyParse("id.namespace == \"myspace\""); verifyParse("id.scheme == \"userdoc\""); @@ -429,28 +465,23 @@ void DocumentSelectParserTest::testParseTerminals() "id.bucket == -9223372036854775566"); verifyParse("id.gid == \"gid(0xd755743aea262650274d70f0)\""); - // Test search column - verifyParse("searchcolumn.10 == 2"); - - // Test other operators + // Test other operators verifyParse("id.scheme = \"*doc\""); verifyParse("testdoctype1.hstringval =~ \"(john|barry|shrek)\""); - // Verify functions + // Verify functions verifyParse("id.hash() == 124"); verifyParse("id.specific.hash() == 124"); verifyParse("testdoctype1.hstringval.lowercase() == \"chang\""); verifyParse("testdoctype1.hstringval.lowercase().hash() == 124"); - verifyFailedParse("testdoctype1 == 8", "ParsingFailedException: Unexpected token" - " at position 13 ('== 8') in query 'testdoctype1 == 8', at fullParse in "); verifyParse("testdoctype1.hintval > now()"); verifyParse("testdoctype1.hintval > now().abs()"); - // Value grouping + // Value grouping verifyParse("(123) < (200)"); verifyParse("(\"hmm\") < (id.scheme)"); - // Arithmetics + // Arithmetics verifyParse("1 + 2 > 1"); verifyParse("1 - 2 > 1"); verifyParse("1 * 2 > 1"); @@ -459,11 +490,11 @@ void DocumentSelectParserTest::testParseTerminals() verifyParse("(1 + 2) * (4 - 2) == 1"); verifyParse("23 + 643 / 34 % 10 > 34"); - // CJK stuff + // CJK stuff verifyParse("testdoctype1.hstringval = \"\xE4\xB8\xBA\xE4\xBB\x80\"", "testdoctype1.hstringval = \"\\xe4\\xb8\\xba\\xe4\\xbb\\x80\""); - // Strange doctype names + // Strange doctype names verifyParse("notandor"); verifyParse("ornotand"); verifyParse("andornot"); @@ -475,16 +506,16 @@ void DocumentSelectParserTest::testParseBranches() { createDocs(); - verifyParse("TRUE or FALSE aNd FALSE oR TRUE"); - verifyParse("TRUE and FALSE or FALSE and TRUE"); - verifyParse("TRUE or FALSE and FALSE or TRUE"); - verifyParse("(TRUE or FALSE) and (FALSE or TRUE)"); + verifyParse("TRUE or FALSE aNd FALSE oR TRUE", "true or false and false or true"); + verifyParse("TRUE and FALSE or FALSE and TRUE", "true and false or false and true"); + verifyParse("TRUE or FALSE and FALSE or TRUE", "true or false and false or true"); + verifyParse("(TRUE or FALSE) and (FALSE or TRUE)", "(true or false) and (false or true)"); verifyParse("true or (not false) and not true"); - // Test number branching with node branches + // Test number branching with node branches verifyParse("((243) < 300 and (\"FOO\").lowercase() == (\"foo\"))"); - // Strange doctype names + // Strange doctype names verifyParse("notandor and ornotand"); verifyParse("ornotand or andornot"); verifyParse("not andornot"); @@ -554,7 +585,7 @@ void DocumentSelectParserTest::testOperators0() std::cerr << ost.str() << "\n"; } // */ - // Check that comparison operators work. + // Check that comparison operators work. PARSE("", *_doc[0], True); PARSE("30 < 10", *_doc[0], False); PARSE("10 < 30", *_doc[0], True); @@ -593,7 +624,7 @@ void DocumentSelectParserTest::testOperators1() { createDocs(); - // Mix of types should within numbers, but otherwise not match + // Mix of types should within numbers, but otherwise not match PARSE("30 < 10.2", *_doc[0], False); PARSE("10.2 < 30", *_doc[0], True); PARSE("30 < \"foo\"", *_doc[0], Invalid); @@ -606,7 +637,7 @@ void DocumentSelectParserTest::testOperators1() PARSE("14.3 == null", *_doc[0], False); PARSE("null = 0", *_doc[0], False); - // Field values + // Field values PARSE("testdoctype1.headerval = 24", *_doc[0], True); PARSE("testdoctype1.headerval = 24", *_doc[1], False); PARSE("testdoctype1.headerval = 13", *_doc[0], False); @@ -625,11 +656,11 @@ void DocumentSelectParserTest::testOperators1() PARSE("testdoctype1.byteweightedset == 7", *_doc[1], False); PARSE("testdoctype1.byteweightedset == 5", *_doc[1], True); - // Document types + // Document types PARSE("testdoctype1", *_doc[0], True); PARSE("testdoctype2", *_doc[0], False); - // Inherited doctypes + // Inherited doctypes PARSE("testdoctype2", *_doc[4], True); PARSE("testdoctype2", *_doc[3], False); PARSE("testdoctype1", *_doc[4], True); @@ -640,7 +671,7 @@ void DocumentSelectParserTest::testOperators2() { createDocs(); - // Id values + // Id values PARSEI("id == \"doc:myspace:anything\"", *_doc[0], True); PARSEI(" iD== \"doc:myspace:anything\" ", *_doc[0], True); PARSEI("id == \"doc:myspa:nything\"", *_doc[0], False); @@ -702,16 +733,13 @@ void DocumentSelectParserTest::testOperators3() PARSEI("id.user = 1234", *_doc[8], True); PARSEI("id.group == \"1234\"", *_doc[8], True); PARSEI("id.group == \"mygroup\"", *_doc[9], True); - - // Searchcolumn policy - PARSE("searchcolumn.10 == 8", *_doc[0], True); } void DocumentSelectParserTest::testOperators4() { createDocs(); - // Branch operators + // Branch operators PARSEI("true and false", *_doc[0], False); PARSEI("true and true", *_doc[0], True); PARSEI("true or false", *_doc[0], True); @@ -723,7 +751,7 @@ void DocumentSelectParserTest::testOperators4() PARSEI("true and not false or false", *_doc[0], True); PARSEI("((243 < 300) and (\"FOO\".lowercase() == \"foo\"))", *_doc[0], True); - // Invalid branching. testdoctype1.content = 1 is invalid + // Invalid branching. testdoctype1.content = 1 is invalid PARSE("testdoctype1.content = 1 and true", *_doc[0], Invalid); PARSE("testdoctype1.content = 1 or true", *_doc[0], True); PARSE("testdoctype1.content = 1 and false", *_doc[0], False); @@ -738,7 +766,7 @@ void DocumentSelectParserTest::testOperators5() { createDocs(); - // Functions + // Functions PARSE("testdoctype1.hstringval.lowercase() == \"Yet\"", *_doc[3], False); PARSE("testdoctype1.hstringval.lowercase() == \"yet\"", *_doc[3], True); PARSE("testdoctype1.hfloatval.lowercase() == \"yet\"", *_doc[3], Invalid); @@ -754,7 +782,7 @@ void DocumentSelectParserTest::testOperators5() PARSE("now() < 1311862500", *_doc[10], False); PARSE("now() > 1611862500", *_doc[10], False); - // Arithmetics + // Arithmetics PARSEI("id.specific.hash() % 10 = 8", *_doc[0], True); PARSEI("id.specific.hash() % 10 = 2", *_doc[0], False); PARSEI("\"foo\" + \"bar\" = \"foobar\"", *_doc[0], True); @@ -767,14 +795,18 @@ void DocumentSelectParserTest::testOperators6() { createDocs(); - // CJK - // Assuming the characters " \ ? * is not used as part of CJK tokens + // CJK + // Assuming the characters " \ ? * is not used as part of CJK tokens PARSE("testdoctype1.content=\"\xE4\xB8\xBA\xE4\xBB\x80\"", *_doc[3], True); PARSE("testdoctype1.content=\"\xE4\xB7\xBA\xE4\xBB\x80\"", *_doc[3], False); - // Structs and arrays + // Structs and arrays PARSE("testdoctype1.mystruct", *_doc[0], False); PARSE("testdoctype1.mystruct", *_doc[1], True); + PARSE("(testdoctype1.mystruct)", *_doc[0], False); + PARSE("(testdoctype1.mystruct)", *_doc[1], True); + PARSE("(((testdoctype1.mystruct)))", *_doc[0], False); + PARSE("(((testdoctype1.mystruct)))", *_doc[1], True); PARSE("testdoctype1.mystruct", *_doc[2], False); PARSE("testdoctype1.mystruct == testdoctype1.mystruct", *_doc[0], True); PARSE("testdoctype1.mystruct == testdoctype1.mystruct", *_doc[1], True); @@ -812,6 +844,7 @@ void DocumentSelectParserTest::testOperators7() PARSE("testdoctype1.structarray", *_doc[1], True); PARSE("testdoctype1.structarray.key == 15", *_doc[1], True); PARSE("testdoctype1.structarray[1].key == 16", *_doc[1], True); + PARSE("testdoctype1.structarray[1].key", *_doc[1], True); // "key is set?" expr PARSE("testdoctype1.structarray[1].key = 16", *_doc[1], True); PARSE("testdoctype1.structarray.value == \"structval1\"", *_doc[0], False); PARSE("testdoctype1.structarray[4].value == \"structval1\"", *_doc[0], False); @@ -952,7 +985,6 @@ namespace { void visitArithmeticValueNode(const select::ArithmeticValueNode &) override {} void visitFunctionValueNode(const select::FunctionValueNode &) override {} void visitIdValueNode(const select::IdValueNode &) override {} - void visitSearchColumnValueNode(const select::SearchColumnValueNode &) override {} void visitFieldValueNode(const select::FieldValueNode &) override {} void visitFloatValueNode(const select::FloatValueNode &) override {} void visitVariableValueNode(const select::VariableValueNode &) override {} @@ -977,15 +1009,14 @@ void DocumentSelectParserTest::testVisitor() TestVisitor v; root->visit(v); + std::string expected = - "OR(CONSTANT(true), " - "AND(DOCTYPE(testdoctype1), " - "AND(OR(NOT(COMPARE(id.user = 12)), " - "COMPARE(testdoctype1.hstringval = \"ola\")), " - "COMPARE(testdoctype1.headerval != null)" - ")" - ")" - ")"; + "OR(CONSTANT(true), " + "AND(AND(DOCTYPE(testdoctype1), " + "OR(NOT(COMPARE(id.user = 12)), " + "COMPARE(testdoctype1.hstringval = \"ola\"))), " + "COMPARE(testdoctype1.headerval != null)))"; + CPPUNIT_ASSERT_EQUAL(expected, v.getVisitString()); } @@ -1093,13 +1124,15 @@ void DocumentSelectParserTest::testDocumentUpdates0() PARSEI("\"\" =~ \"\"", *_update[0], True); PARSEI("30 = 10", *_update[0], False); PARSEI("30 = 30", *_update[0], True); + PARSEI("(30 = 10)", *_update[0], False); + PARSEI("(30 = 30)", *_update[0], True); } void DocumentSelectParserTest::testDocumentUpdates1() { createDocs(); - // Mix of types should within numbers, but otherwise not match + // Mix of types should within numbers, but otherwise not match PARSEI("30 < 10.2", *_update[0], False); PARSEI("10.2 < 30", *_update[0], True); PARSEI("30 < \"foo\"", *_update[0], Invalid); @@ -1112,17 +1145,18 @@ void DocumentSelectParserTest::testDocumentUpdates1() PARSEI("14.3 == null", *_update[0], False); PARSEI("null = 0", *_update[0], False); - // Field values + // Field values PARSE("testdoctype1.headerval = 24", *_update[0], Invalid); PARSE("testdoctype1.hfloatval = 2.0", *_update[0], Invalid); PARSE("testdoctype1.content = \"bar\"", *_update[0], Invalid); PARSE("testdoctype1.hstringval == testdoctype1.content", *_update[0], Invalid); - // Document types + // Document types PARSE("testdoctype1", *_update[0], True); + PARSE("(testdoctype1)", *_update[0], True); PARSE("testdoctype2", *_update[0], False); - // Inherited doctypes + // Inherited doctypes PARSE("testdoctype2", *_update[4], True); PARSE("testdoctype2", *_update[3], False); PARSE("testdoctype1", *_update[4], True); @@ -1133,7 +1167,7 @@ void DocumentSelectParserTest::testDocumentUpdates2() { createDocs(); - // Id values + // Id values PARSEI("id == \"doc:myspace:anything\"", *_update[0], True); PARSEI(" iD== \"doc:myspace:anything\" ", *_update[0], True); PARSEI("id == \"doc:myspa:nything\"", *_update[0], False); @@ -1159,7 +1193,7 @@ void DocumentSelectParserTest::testDocumentUpdates3() { createDocs(); - // Branch operators + // Branch operators PARSEI("true and false", *_update[0], False); PARSEI("true and true", *_update[0], True); PARSEI("true or false", *_update[0], True); @@ -1171,7 +1205,7 @@ void DocumentSelectParserTest::testDocumentUpdates3() PARSEI("true and not false or false", *_update[0], True); PARSEI("((243 < 300) and (\"FOO\".lowercase() == \"foo\"))", *_update[0], True); - // Invalid branching. testdoctype1.content = 1 is invalid + // Invalid branching. testdoctype1.content = 1 is invalid PARSE("testdoctype1.content = 1 and true", *_update[0], Invalid); PARSE("testdoctype1.content = 1 or true", *_update[0], True); PARSE("testdoctype1.content = 1 and false", *_update[0], False); @@ -1186,7 +1220,7 @@ void DocumentSelectParserTest::testDocumentUpdates4() { createDocs(); - // Functions + // Functions PARSEI("\"bar\".hash() == -2012135647395072713", *_update[0], True); PARSEI("\"bar\".hash().abs() == 2012135647395072713", *_update[0], True); PARSEI("null.hash() == 123", *_update[0], Invalid); @@ -1195,7 +1229,7 @@ void DocumentSelectParserTest::testDocumentUpdates4() PARSEI("\"foo\".hash() == 123", *_update[0], False); PARSEI("(234).hash() == 123", *_update[0], False); - // Arithmetics + // Arithmetics PARSEI("id.specific.hash() % 10 = 8", *_update[0], True); PARSEI("id.specific.hash() % 10 = 2", *_update[0], False); PARSEI("\"foo\" + \"bar\" = \"foobar\"", *_update[0], True); @@ -1221,7 +1255,7 @@ void DocumentSelectParserTest::testUtf8() } std::unique_ptr<select::FieldValueNode> -DocumentSelectParserTest::parseFieldValue(const std::string expression) { +DocumentSelectParserTest::parseFieldValue(const std::string& expression) { return std::unique_ptr<select::FieldValueNode>(dynamic_cast<select::FieldValueNode *>( dynamic_cast<const select::Compare &>(*_parser->parse(expression)).getLeft().clone().release())); } @@ -1246,4 +1280,296 @@ void DocumentSelectParserTest::testThatComplexFieldValuesHaveCorrectFieldNames() parseFieldValue("testdoctype1.headerval.meow.meow{test}")->getRealFieldName()); } +namespace { + +class OperatorVisitor : public select::Visitor { +private: + std::ostringstream data; +public: + void visitConstant(const select::Constant& node) override { + data << node; + } + + void + visitInvalidConstant(const select::InvalidConstant& node) override { + (void) node; + assert(false); + } + + void visitDocumentType(const select::DocType& node) override { + data << "(DOCTYPE " << node << ")"; + } + + void visitComparison(const select::Compare& node) override { + data << '(' << node.getOperator() << ' '; + node.getLeft().visit(*this); + data << ' '; + node.getRight().visit(*this); + data << ')'; + } + + void visitAndBranch(const select::And& node) override { + data << "(AND "; + node.getLeft().visit(*this); + data << " "; + node.getRight().visit(*this); + data << ")"; + } + + void visitOrBranch(const select::Or& node) override { + data << "(OR "; + node.getLeft().visit(*this); + data << " "; + node.getRight().visit(*this); + data << ")"; + } + + void visitNotBranch(const select::Not& node) override { + data << "(NOT "; + node.getChild().visit(*this); + data << ")"; + } + + void visitArithmeticValueNode(const select::ArithmeticValueNode& node) override { + data << '(' << node.getOperatorName() << ' '; + node.getLeft().visit(*this); + data << ' '; + node.getRight().visit(*this); + data << ')'; + } + void visitFunctionValueNode(const select::FunctionValueNode& node) override { + data << '(' << node.getFunctionName() << ' '; + node.getChild().visit(*this); + data << ')'; + } + void visitIdValueNode(const select::IdValueNode& node) override { + data << "(ID " << node.toString() << ')'; + } + void visitFieldValueNode(const select::FieldValueNode& node) override { + data << "(FIELD " << node.getDocType() << ' ' << node.getFieldName() << ')'; + } + void visitFloatValueNode(const select::FloatValueNode& node) override { + data << node.getValue(); + } + void visitVariableValueNode(const select::VariableValueNode& node) override { + data << "(VAR " << node.getVariableName() << ')'; + } + void visitIntegerValueNode(const select::IntegerValueNode& node) override { + data << node.getValue(); + } + void visitCurrentTimeValueNode(const select::CurrentTimeValueNode&) override {} + void visitStringValueNode(const select::StringValueNode& str) override { + data << '"' << str.getValue() << '"'; + } + void visitNullValueNode(const select::NullValueNode&) override { + data << "null"; + } + void visitInvalidValueNode(const select::InvalidValueNode&) override { + data << "INVALID!"; + } + + std::string visit_string() { return data.str(); } +}; + +template <typename NodeType> +std::string node_to_string(const NodeType& node) { + OperatorVisitor v; + node.visit(v); + return v.visit_string(); +} + +} + +std::string DocumentSelectParserTest::parse_to_tree(const std::string& str) { + std::unique_ptr<select::Node> root(_parser->parse(str)); + return node_to_string(*root); +} + +void DocumentSelectParserTest::test_operator_precedence() { + createDocs(); + using namespace std::string_literals; + + CPPUNIT_ASSERT_EQUAL("(AND true false)"s, parse_to_tree("true and false")); + CPPUNIT_ASSERT_EQUAL("(AND (NOT false) true)"s, parse_to_tree("not false and true")); + CPPUNIT_ASSERT_EQUAL("(NOT (AND false true))"s, parse_to_tree("not (false and true)")); + CPPUNIT_ASSERT_EQUAL("(NOT (DOCTYPE testdoctype1))"s, parse_to_tree("not testdoctype1")); + CPPUNIT_ASSERT_EQUAL("(NOT (DOCTYPE (testdoctype1)))"s, parse_to_tree("not (testdoctype1)")); + CPPUNIT_ASSERT_EQUAL("(NOT (DOCTYPE (testdoctype1)))"s, parse_to_tree("(not (testdoctype1))")); + CPPUNIT_ASSERT_EQUAL("(OR (== 1 2) (== 3 4))"s, parse_to_tree("1==2 or 3==4")); + CPPUNIT_ASSERT_EQUAL("(!= (+ (+ 1 2) 3) 0)"s, parse_to_tree("1+2+3 != 0")); + CPPUNIT_ASSERT_EQUAL("(!= (+ (+ 1.1 2.2) 3.3) 4.4)"s, parse_to_tree("1.1+2.2+3.3 != 4.4")); + CPPUNIT_ASSERT_EQUAL("(!= (- (- 1 2) 3) 0)"s, parse_to_tree("1-2-3 != 0")); + CPPUNIT_ASSERT_EQUAL("(!= (+ (+ 1 2) 3) 0)"s, parse_to_tree("1 + 2 + 3 != 0")); + CPPUNIT_ASSERT_EQUAL("(!= (+ 1 (* 2 3)) 0)"s, parse_to_tree("1 + 2 * 3 != 0")); + CPPUNIT_ASSERT_EQUAL("(!= (- (/ (* 1 2) 3) 4) 0)"s, parse_to_tree("1 * 2 / 3 - 4 != 0")); + CPPUNIT_ASSERT_EQUAL("(!= (/ (* 1 2) (- 3 4)) 0)"s, parse_to_tree("1 * 2 / (3 - 4) != 0")); + CPPUNIT_ASSERT_EQUAL("(OR (AND true (NOT (== 1 2))) false)"s, + parse_to_tree("true and not 1 == 2 or false")); + CPPUNIT_ASSERT_EQUAL("(AND (AND (AND (< 1 2) (> 3 4)) (<= 5 6)) (>= 7 8))"s, + parse_to_tree("1 < 2 and 3 > 4 and 5 <= 6 and 7 >= 8")); + CPPUNIT_ASSERT_EQUAL("(OR (AND (AND (< 1 2) (> 3 4)) (<= 5 6)) (>= 7 8))"s, + parse_to_tree("1 < 2 and 3 > 4 and 5 <= 6 or 7 >= 8")); + CPPUNIT_ASSERT_EQUAL("(OR (AND (< 1 2) (> 3 4)) (AND (<= 5 6) (>= 7 8)))"s, + parse_to_tree("1 < 2 and 3 > 4 or 5 <= 6 and 7 >= 8")); + // Unary plus is simply ignored by the parser. + CPPUNIT_ASSERT_EQUAL("(== 1 -2)"s, parse_to_tree("+1==-2")); + CPPUNIT_ASSERT_EQUAL("(== 1.23 -2.56)"s, parse_to_tree("+1.23==-2.56")); + CPPUNIT_ASSERT_EQUAL("(== (+ 1 2) (- 3 -4))"s, parse_to_tree("1 + +2==3 - -4")); + CPPUNIT_ASSERT_EQUAL("(== (+ 1 2) (- 3 -4))"s, parse_to_tree("1++2==3--4")); + + // Due to the way parentheses are handled by the AST, ((foo)) always gets + // reduced down to (foo). + CPPUNIT_ASSERT_EQUAL("(DOCTYPE (testdoctype1))"s, parse_to_tree("(((testdoctype1)))")); + CPPUNIT_ASSERT_EQUAL("(AND (DOCTYPE (testdoctype1)) (DOCTYPE (testdoctype2)))"s, + parse_to_tree("((((testdoctype1))) and ((testdoctype2)))")); + + CPPUNIT_ASSERT_EQUAL("(== (ID id) \"foo\")"s, parse_to_tree("id == 'foo'")); + CPPUNIT_ASSERT_EQUAL("(== (ID id.group) \"foo\")"s, parse_to_tree("id.group == 'foo'")); + // id_spec function apply + CPPUNIT_ASSERT_EQUAL("(== (hash (ID id)) 12345)"s, parse_to_tree("id.hash() == 12345")); + // Combination of id_spec function apply and arith_expr function apply + CPPUNIT_ASSERT_EQUAL("(== (abs (hash (ID id))) 12345)"s, parse_to_tree("id.hash().abs() == 12345")); +} + +void DocumentSelectParserTest::test_token_used_as_ident_preserves_casing() { + createDocs(); + using namespace std::string_literals; + + // TYPE, SCHEME, ORDER etc are tokens that may also be used as identifiers + // without introducing parsing ambiguities. In this context their original + // casing should be preserved. + CPPUNIT_ASSERT_EQUAL("(== (VAR Type) 123)"s, parse_to_tree("$Type == 123")); + CPPUNIT_ASSERT_EQUAL("(== (VAR giD) 123)"s, parse_to_tree("$giD == 123")); + CPPUNIT_ASSERT_EQUAL("(== (VAR ORDER) 123)"s, parse_to_tree("$ORDER == 123")); +} + +void DocumentSelectParserTest::test_ambiguous_field_spec_expression_is_handled_correctly() { + createDocs(); + using namespace std::string_literals; + // In earlier revisions of LR(1)-grammar, this triggered a reduce/reduce conflict between + // logical_expr and arith_expr for the sequence '(' field_spec ')', which failed to + // parse in an expected manner. Test that we don't get regressions here. + CPPUNIT_ASSERT_EQUAL("(!= (FIELD testdoctype1 foo) null)"s, parse_to_tree("(testdoctype1.foo)")); + CPPUNIT_ASSERT_EQUAL("(AND (!= (FIELD testdoctype1 foo) null) (!= (FIELD testdoctype1 bar) null))"s, + parse_to_tree("(testdoctype1.foo) AND (testdoctype1.bar)")); +} + +void DocumentSelectParserTest::test_can_build_field_value_from_field_expr_node() { + using select::FieldExprNode; + { + // Simple field expression + auto lhs = std::make_unique<FieldExprNode>("mydoctype"); + auto root = std::make_unique<FieldExprNode>(std::move(lhs), "foo"); + auto fv = root->convert_to_field_value(); + CPPUNIT_ASSERT_EQUAL(vespalib::string("mydoctype"), fv->getDocType()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo"), fv->getFieldName()); + } + { + // Nested field expression + auto lhs1 = std::make_unique<FieldExprNode>("mydoctype"); + auto lhs2 = std::make_unique<FieldExprNode>(std::move(lhs1), "foo"); + auto root = std::make_unique<FieldExprNode>(std::move(lhs2), "bar"); + auto fv = root->convert_to_field_value(); + CPPUNIT_ASSERT_EQUAL(vespalib::string("mydoctype"), fv->getDocType()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo.bar"), fv->getFieldName()); + } +} + +void DocumentSelectParserTest::test_can_build_function_call_from_field_expr_node() { + using select::FieldExprNode; + { + // doctype.foo.lowercase() + // Note that calling lowercase() directly on the doctype is not supported + // (see test_function_call_on_doctype_throws_exception) + auto lhs1 = std::make_unique<FieldExprNode>("mydoctype"); + auto lhs2 = std::make_unique<FieldExprNode>(std::move(lhs1), "foo"); + auto root = std::make_unique<FieldExprNode>(std::move(lhs2), "lowercase"); + auto func = root->convert_to_function_call(); + CPPUNIT_ASSERT_EQUAL(vespalib::string("lowercase"), func->getFunctionName()); + // TODO vespalib::string? + CPPUNIT_ASSERT_EQUAL(std::string("(FIELD mydoctype foo)"), node_to_string(func->getChild())); + } +} + +void DocumentSelectParserTest::test_function_call_on_doctype_throws_exception() { + using select::FieldExprNode; + auto lhs = std::make_unique<FieldExprNode>("mydoctype"); + auto root = std::make_unique<FieldExprNode>(std::move(lhs), "lowercase"); + try { + root->convert_to_function_call(); + } catch (const vespalib::IllegalArgumentException& e) { + CPPUNIT_ASSERT_EQUAL(vespalib::string("Cannot call function 'lowercase' directly on document type"), + e.getMessage()); + } +} + +namespace { + +void check_parse_i64(vespalib::stringref str, bool expect_ok, int64_t expected_output) { + int64_t out = 0; + bool ok = select::util::parse_i64(str.data(), str.size(), out); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Parsing did not returned expected success status for i64 input " + str, expect_ok, ok); + if (expect_ok) { + CPPUNIT_ASSERT_EQUAL_MESSAGE("Parse output not as expected for i64 input " + str, expected_output, out); + } +} + +void check_parse_hex_i64(vespalib::stringref str, bool expect_ok, int64_t expected_output) { + int64_t out = 0; + bool ok = select::util::parse_hex_i64(str.data(), str.size(), out); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Parsing did not returned expected success status for hex i64 input " + str, expect_ok, ok); + if (expect_ok) { + CPPUNIT_ASSERT_EQUAL_MESSAGE("Parse output not as expected for hex i64 input " + str, expected_output, out); + } +} + +void check_parse_double(vespalib::stringref str, bool expect_ok, double expected_output) { + double out = 0; + bool ok = select::util::parse_double(str.data(), str.size(), out); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Parsing did not returned expected success status for hex i64 input " + str, expect_ok, ok); + if (expect_ok) { + CPPUNIT_ASSERT_EQUAL_MESSAGE("Parse output not as expected for double input " + str, expected_output, out); + } +} + +} + +void DocumentSelectParserTest::test_parse_utilities_handle_well_formed_input() { + check_parse_i64("0", true, 0); + check_parse_i64("1", true, 1); + check_parse_i64("9223372036854775807", true, INT64_MAX); + + // Note: 0x prefix is _not_ included + check_parse_hex_i64("0", true, 0); + check_parse_hex_i64("1", true, 1); + check_parse_hex_i64("f", true, 15); + check_parse_hex_i64("F", true, 15); + check_parse_hex_i64("ffffffff", true, UINT32_MAX); + check_parse_hex_i64("7FFFFFFFFFFFFFFF", true, INT64_MAX); + // We actually parse as u64 internally, then convert + check_parse_hex_i64("ffffffffffffffff", true, -1); + + check_parse_double("1.0", true, 1.0); + check_parse_double("1.", true, 1.0); + check_parse_double("1.79769e+308", true, 1.79769e+308); // DBL_MAX +} + +void DocumentSelectParserTest::test_parse_utilities_handle_malformed_input() { + check_parse_i64("9223372036854775808", false, 0); // INT64_MAX + 1 + check_parse_i64("18446744073709551615", false, 0); // UINT64_MAX + check_parse_i64("", false, 0); + check_parse_i64("bjarne", false, 0); + check_parse_i64("1x", false, 0); + + check_parse_hex_i64("", false, 0); + check_parse_hex_i64("g", false, 0); + check_parse_hex_i64("0x1", false, 0); + check_parse_hex_i64("ffffffffffffffff1", false, 0); + + check_parse_double("1.x", false, 0.0); + // TODO double outside representable range returns Inf, but we probably would + // like this to trigger a parse failure? + check_parse_double("1.79769e+309", true, std::numeric_limits<double>::infinity()); +} + } // document diff --git a/document/src/vespa/document/bucket/bucketselector.cpp b/document/src/vespa/document/bucket/bucketselector.cpp index 5ded691269a..ceb231a483c 100644 --- a/document/src/vespa/document/bucket/bucketselector.cpp +++ b/document/src/vespa/document/bucket/bucketselector.cpp @@ -137,27 +137,6 @@ using namespace document::select; } } - void compare(const select::SearchColumnValueNode& node, - const select::ValueNode& valnode, - const select::Operator& op) { - if (op == FunctionOperator::EQ || op == document::select::GlobOperator::GLOB) { - int bucketCount = 1 << 16; - const IntegerValueNode* val( - dynamic_cast<const IntegerValueNode*>(&valnode)); - - int64_t rval = val->getValue(); - - for (int i = 0; i < bucketCount; i++) { - int64_t column = node.getValue(BucketId(16, i)); - if (column == rval) { - _buckets.push_back(BucketId(16, i)); - } - } - - _unknown = false; - } - } - void visitComparison(const document::select::Compare& node) override { if (node.getOperator() != document::select::FunctionOperator::EQ && node.getOperator() != document::select::GlobOperator::GLOB) @@ -166,12 +145,8 @@ using namespace document::select; } const IdValueNode* lid(dynamic_cast<const IdValueNode*>( &node.getLeft())); - const SearchColumnValueNode* sc(dynamic_cast<const SearchColumnValueNode*>( - &node.getLeft())); if (lid) { compare(*lid, node.getRight(), node.getOperator()); - } else if (sc) { - compare(*sc, node.getRight(), node.getOperator()); } else { const IdValueNode* rid(dynamic_cast<const IdValueNode*>( &node.getRight())); @@ -187,7 +162,6 @@ using namespace document::select; void visitArithmeticValueNode(const ArithmeticValueNode &) override {} void visitFunctionValueNode(const FunctionValueNode &) override {} void visitIdValueNode(const IdValueNode &) override {} - void visitSearchColumnValueNode(const SearchColumnValueNode &) override {} void visitFieldValueNode(const FieldValueNode &) override {} void visitFloatValueNode(const FloatValueNode &) override {} void visitVariableValueNode(const VariableValueNode &) override {} diff --git a/document/src/vespa/document/select/.gitignore b/document/src/vespa/document/select/.gitignore index 5f004816692..919eb5c7ca9 100644 --- a/document/src/vespa/document/select/.gitignore +++ b/document/src/vespa/document/select/.gitignore @@ -2,3 +2,6 @@ Makefile .depend* .*.swp *.So +*.hxx +*.cxx +*.hh diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt index 0e94fa0f530..eba5ddde40c 100644 --- a/document/src/vespa/document/select/CMakeLists.txt +++ b/document/src/vespa/document/select/CMakeLists.txt @@ -1,4 +1,14 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +find_package(BISON REQUIRED) +find_package(FLEX REQUIRED) + +BISON_TARGET(DocSelParser grammar/parser.yy ${CMAKE_CURRENT_BINARY_DIR}/parser.cxx) +FLEX_TARGET(DocSelLexer grammar/lexer.ll ${CMAKE_CURRENT_BINARY_DIR}/lexer.cxx) + +ADD_FLEX_BISON_DEPENDENCY(DocSelLexer DocSelParser) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + vespa_add_library(document_select OBJECT SOURCES bodyfielddetector.cpp @@ -13,7 +23,6 @@ vespa_add_library(document_select OBJECT operator.cpp orderingselector.cpp orderingspecification.cpp - parser.cpp result.cpp resultset.cpp resultlist.cpp @@ -22,6 +31,11 @@ vespa_add_library(document_select OBJECT value.cpp valuenode.cpp valuenodes.cpp + parser.cpp + parse_utils.cpp + parsing_failed_exception.cpp + ${BISON_DocSelParser_OUTPUTS} + ${FLEX_DocSelLexer_OUTPUTS} AFTER document_documentconfig ) diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp index 7f6ad252471..5b28fad0df8 100644 --- a/document/src/vespa/document/select/branch.cpp +++ b/document/src/vespa/document/select/branch.cpp @@ -9,7 +9,7 @@ namespace document { namespace select { And::And(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name) - : Branch(name ? name : "AND"), + : Branch(name ? name : "and"), _left(std::move(left)), _right(std::move(right)) { @@ -55,7 +55,7 @@ And::trace(const Context& context, std::ostream& out) const } Or::Or(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name) - : Branch(name ? name : "OR"), + : Branch(name ? name : "or"), _left(std::move(left)), _right(std::move(right)) { @@ -101,7 +101,7 @@ Or::trace(const Context& context, std::ostream& out) const } Not::Not(std::unique_ptr<Node> child, const char* name) - : Branch(name ? name : "NOT"), + : Branch(name ? name : "not"), _child(std::move(child)) { assert(_child.get()); diff --git a/document/src/vespa/document/select/cloningvisitor.cpp b/document/src/vespa/document/select/cloningvisitor.cpp index 4011cbdeea1..d695e3ec83d 100644 --- a/document/src/vespa/document/select/cloningvisitor.cpp +++ b/document/src/vespa/document/select/cloningvisitor.cpp @@ -162,7 +162,7 @@ CloningVisitor::visitConstant(const Constant &expr) _priority = ConstPriority; bool val = expr.getConstantValue(); _resultSet.add(val ? Result::True : Result::False); - _node.reset(new Constant(val ? "true" : "false")); + _node.reset(new Constant(val)); } @@ -199,16 +199,6 @@ CloningVisitor::visitIdValueNode(const IdValueNode &expr) void -CloningVisitor::visitSearchColumnValueNode(const SearchColumnValueNode &expr) -{ - _constVal = false; - ++_fieldNodes; // needs document id, thus needs document - _valueNode = expr.clone(); - _priority = SearchColPriority; -} - - -void CloningVisitor::visitFieldValueNode(const FieldValueNode &expr) { _constVal = false; diff --git a/document/src/vespa/document/select/cloningvisitor.h b/document/src/vespa/document/select/cloningvisitor.h index ff74af1201d..2c5f94c20a4 100644 --- a/document/src/vespa/document/select/cloningvisitor.h +++ b/document/src/vespa/document/select/cloningvisitor.h @@ -60,7 +60,6 @@ public: void visitInvalidConstant(const InvalidConstant &expr) override; void visitDocumentType(const DocType &expr) override; void visitIdValueNode(const IdValueNode &expr) override; - void visitSearchColumnValueNode(const SearchColumnValueNode &expr) override; void visitFieldValueNode(const FieldValueNode &expr) override; void visitFloatValueNode(const FloatValueNode &expr) override; void visitVariableValueNode(const VariableValueNode &expr) override; diff --git a/document/src/vespa/document/select/constant.cpp b/document/src/vespa/document/select/constant.cpp index 02821180337..8d1445f40a0 100644 --- a/document/src/vespa/document/select/constant.cpp +++ b/document/src/vespa/document/select/constant.cpp @@ -7,28 +7,10 @@ namespace document::select { -Constant::Constant(const vespalib::stringref & value) - : Node(value), - _value(false) +Constant::Constant(bool value) + : Node(value ? "true" : "false"), // TODO remove required name from Node + _value(value) { - if (value.size() == 4 && - (value[0] & 0xdf) == 'T' && - (value[1] & 0xdf) == 'R' && - (value[2] & 0xdf) == 'U' && - (value[3] & 0xdf) == 'E') - { - _value = true; - } else if (value.size() == 5 && - (value[0] & 0xdf) == 'F' && - (value[1] & 0xdf) == 'A' && - (value[2] & 0xdf) == 'L' && - (value[3] & 0xdf) == 'S' && - (value[4] & 0xdf) == 'E') - { - _value = false; - } else { - assert(false); - } } ResultList diff --git a/document/src/vespa/document/select/constant.h b/document/src/vespa/document/select/constant.h index 08be5c95ec7..46a98ed7eaa 100644 --- a/document/src/vespa/document/select/constant.h +++ b/document/src/vespa/document/select/constant.h @@ -5,9 +5,8 @@ * * @brief Class describing a constant in the select tree. * - * @author H�kon Humberset + * @author Håkon Humberset * @date 2005-06-07 - * @version $Id$ */ #pragma once @@ -23,7 +22,7 @@ private: bool _value; public: - explicit Constant(const vespalib::stringref & value); + explicit Constant(bool value); ResultList contains(const Context&) const override { return ResultList(Result::get(_value)); @@ -32,8 +31,8 @@ public: ResultList trace(const Context&, std::ostream& trace) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; void visit(Visitor& v) const override; - bool getConstantValue() const { return _value; } - Node::UP clone() const override { return wrapParens(new Constant(_name)); } + bool getConstantValue() const noexcept { return _value; } + Node::UP clone() const override { return wrapParens(new Constant(_value)); } }; diff --git a/document/src/vespa/document/select/gid_filter.cpp b/document/src/vespa/document/select/gid_filter.cpp index ce3045564ba..71a57a0886a 100644 --- a/document/src/vespa/document/select/gid_filter.cpp +++ b/document/src/vespa/document/select/gid_filter.cpp @@ -22,7 +22,6 @@ struct NoOpVisitor : Visitor { void visitArithmeticValueNode(const ArithmeticValueNode&) override {} void visitFunctionValueNode(const FunctionValueNode&) override {} void visitIdValueNode(const IdValueNode&) override {} - void visitSearchColumnValueNode(const SearchColumnValueNode&) override {} void visitFieldValueNode(const FieldValueNode&) override {} void visitFloatValueNode(const FloatValueNode&) override {} void visitVariableValueNode(const VariableValueNode&) override {} diff --git a/document/src/vespa/document/select/grammar/lexer.ll b/document/src/vespa/document/select/grammar/lexer.ll new file mode 100644 index 00000000000..8cd5638c122 --- /dev/null +++ b/document/src/vespa/document/select/grammar/lexer.ll @@ -0,0 +1,182 @@ + /* Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. */ + + /* We use the .*xx-suffix to denote a build-time generated file */ +%option outfile="lexer.cxx" +%option header-file="lexer.hxx" + +%option c++ + /* Uncomment to enable debug tracing of parsing */ + /* %option debug */ +%option 8bit warn nodefault +%option noyywrap nounput +%option yyclass="document::select::DocSelScanner" + + /* Used to track source locations, see https://github.com/bingmann/flex-bison-cpp-example/blob/master/src/scanner.ll */ +%{ +#define YY_USER_ACTION yyloc->columns(yyleng); +%} + +%{ + +#include "parser.hxx" +#include <vespa/document/select/scanner.h> +#include <vespa/document/select/parse_utils.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/stllike/string.h> +#include <string> +#include <cstdlib> + +#undef YY_DECL +#define YY_DECL int document::select::DocSelScanner::yylex( \ + document::select::DocSelParser::semantic_type* yylval, \ + document::select::DocSelParser::location_type* yyloc) + +using token = document::select::DocSelParser::token; +using string = vespalib::string; + +// Inspired by https://coldfix.eu/2015/05/16/bison-c++11/ + +#define YIELD_TOKEN(name, field_name, value) \ + yylval->field_name = value; \ + return token::T_##name; + +#define INT_TOKEN(name, value) YIELD_TOKEN(name, i64_val, value) +#define STRING_TOKEN(name) YIELD_TOKEN(name, string_val, new string(yytext, yyleng)) +#define CONST_STR_TOKEN(name, value) YIELD_TOKEN(name, const_str_val, value) +#define TAGGED_TOKEN INT_TOKEN + +#define NAMED_TOKEN(name) return token::T_##name; + +%} + + /* Lexer fragments, used as part of token patterns */ + +SIGN [+-] +DECIMAL [0-9]+ +HEXDIGIT [0-9a-fA-F] +HEX 0[xX]{HEXDIGIT}{1,16} +OCTAL 0[0-7]* +EXPONENT [eE][+-]?[0-9]+ +IDCHARS [a-zA-Z_][a-zA-Z_0-9_]* +WS [ \f\r\t] + + /* It is weird that you can't do \' inside "" and vice versa, but that's the StringUtil::unescape logic today... */ +DQ_STRING \"(\\([\\tnfr"]|x{HEXDIGIT}{2})|[^"\\])*\" +SQ_STRING \'(\\([\\tnfr']|x{HEXDIGIT}{2})|[^'\\])*\' + +%% + + /* Code to take place at the beginning of yylex() */ +%{ + // TODO move to YY_USER_ACTION instead? + yyloc->step(); +%} + + /* TODO support length suffixes? supported in JavaCC grammar, but not in legacy Spirit grammar... */ +{HEX} { + // TODO replace with std::from_string() once compiler support is there + if (!util::parse_hex_i64(yytext + 2, yyleng - 2, yylval->i64_val)) { // Skip 0[xX] prefix + throw_parser_syntax_error(*yyloc, "Not a valid 64-bit hex integer: " + std::string(yytext, yyleng)); + } + return token::T_INTEGER; +} + + /* Sign is handled explicitly in the parser to avoid lexing ambiguities for expressions such as "1 -2" */ +{DECIMAL} { + if (!util::parse_i64(yytext, yyleng, yylval->i64_val)) { + throw_parser_syntax_error(*yyloc, "Not a valid signed 64-bit integer: " + std::string(yytext, yyleng)); + } + return token::T_INTEGER; +} + + /* + * We use a strict definition of floats when lexing, i.e. we require a dot + * in order to remove ambiguities with the base 10 integer token. + */ +[0-9]+(\.[0-9]*){EXPONENT}?[fFdD]? { + if (!util::parse_double(yytext, yyleng, yylval->double_val)) { + throw_parser_syntax_error(*yyloc, "Not a valid floating point number: " + std::string(yytext, yyleng)); + } + return token::T_FLOAT; +} + +({DQ_STRING}|{SQ_STRING}) { + // Always slice off start and end quote chars + yylval->string_val = new string(yytext + 1, yyleng - 2); + return token::T_STRING; +} + + /* FIXME this is a syntactic hack to "flatten" fieldpath map and array lookups into a single token + rather than match these structurally in the parser itself. This is due to the way fieldpaths + are handled in the legacy AST (i.e. as strings, not structures), and this must be changed first + before we can fix this. */ + /* Field path expressions do not support any other escapes than double quote char */ + /* TODO {WS} does not include newline, do we need to support that here? */ +\{{WS}*($?{IDCHARS}|{DECIMAL}|\"([^\\\"]|\\\")*\"){WS}*\} STRING_TOKEN(FP_MAP_LOOKUP) +\[{WS}*(${IDCHARS}|{DECIMAL}){WS}*\] STRING_TOKEN(FP_ARRAY_LOOKUP) + + /* Primary tokens are case insensitive */ +(?i:"id") NAMED_TOKEN(ID) +(?i:"null") NAMED_TOKEN(NULL) +(?i:"true") NAMED_TOKEN(TRUE) +(?i:"false") NAMED_TOKEN(FALSE) +(?i:"and") NAMED_TOKEN(AND) +(?i:"or") NAMED_TOKEN(OR) +(?i:"not") NAMED_TOKEN(NOT) + + /* We expose the verbatim input as the token value, as these may also be used for identifiers... */ +(?i:"user") STRING_TOKEN(USER) +(?i:"group") STRING_TOKEN(GROUP) +(?i:"scheme") STRING_TOKEN(SCHEME) +(?i:"namespace") STRING_TOKEN(NAMESPACE) +(?i:"specific") STRING_TOKEN(SPECIFIC) +(?i:"bucket") STRING_TOKEN(BUCKET) +(?i:"gid") STRING_TOKEN(GID) +(?i:"type") STRING_TOKEN(TYPE) +(?i:"order") STRING_TOKEN(ORDER) + +"now\(\)" NAMED_TOKEN(NOW_FUNC) /* This _is_ case-sensitive in the legacy parser */ + + /* Binary operators */ + /* TODO INT_TOKEN with code directly from selection operator node? Or direct operator object ptr? */ +"=" NAMED_TOKEN(GLOB) +"=~" NAMED_TOKEN(REGEX) +"==" NAMED_TOKEN(EQ) +"!=" NAMED_TOKEN(NE) +">=" NAMED_TOKEN(GE) +"<=" NAMED_TOKEN(LE) +">" NAMED_TOKEN(GT) +"<" NAMED_TOKEN(LT) + +"$" NAMED_TOKEN(DOLLAR) +"." NAMED_TOKEN(DOT) +"(" NAMED_TOKEN(LPAREN) +")" NAMED_TOKEN(RPAREN) +"," NAMED_TOKEN(COMMA) +"+" NAMED_TOKEN(PLUS) +"-" NAMED_TOKEN(MINUS) +"*" NAMED_TOKEN(MULTIPLY) +"/" NAMED_TOKEN(DIVIDE) +"%" NAMED_TOKEN(MODULO) + +{IDCHARS} STRING_TOKEN(IDENTIFIER) + +\n { + yyloc->lines(yyleng); + yyloc->step(); + return yytext[0]; +} + +{WS} { + yyloc->step(); +} + + /* + * Everything that hasn't already matched is an error. Throw exception immediately with the exact + * char to avoid getting auto-generated error messages with "unexpected $undefined" due to the + * resulting token not matching any existing, explicitly named tokens. + */ +. { throw_parser_syntax_error(*yyloc, "Unexpected character: '" + StringUtil::escape(vespalib::string(yytext, 1)) + "'"); } + +%% + diff --git a/document/src/vespa/document/select/grammar/parser.yy b/document/src/vespa/document/select/grammar/parser.yy new file mode 100644 index 00000000000..baf987355c9 --- /dev/null +++ b/document/src/vespa/document/select/grammar/parser.yy @@ -0,0 +1,374 @@ + /* Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. */ + +%output "parser.cxx" +%defines "parser.hxx" + + /* Skeleton implementation included as part of the generated source. Note: _not_ covered by the GPL. */ +%skeleton "lalr1.cc" + +%require "3.0" + + /* Uncomment to enable debugging of lexer invocations */ + /*%debug*/ + +%locations + +%define parse.error verbose +%define parse.assert + +%define api.token.prefix {T_} +%define api.namespace {document::select} +%define parser_class_name {DocSelParser} + + /* + * Due to current Bison variant support not being move-enabled (and our AST ptrs being move-only), + * we have to use good old POD unions for our rule results. Note that we have to use %destructor + * for all ptrs to ensure cleanup. + */ +%union { + int64_t i64_val; + double double_val; + const char* const_str_val; + vespalib::string* string_val; + Constant* constant_node; + ValueNode* value_node; + FieldExprNode* field_expr_node; + Node* abstract_node; +} + +%token END 0 "end of input" +%token NULL +%token TRUE +%token FALSE +%token AND +%token OR +%token NOT + + /* Specify aliases for several tokens for ease of use and better error reporting */ +%token GLOB "=" +%token REGEX "=~" +%token EQ "==" +%token NE "!=" +%token GE ">=" +%token LE "<=" +%token GT ">" +%token LT "<" +%token ID +%token NOW_FUNC + + /* + * Tokens that we only mention by alias in the grammar rules, but which we define + * explicitly to improve error reporting + */ +%token DOLLAR "$" +%token DOT "." +%token LPAREN "(" +%token RPAREN ")" +%token COMMA "," +%token PLUS "+" +%token MINUS "-" +%token MULTIPLY "*" +%token DIVIDE "/" +%token MODULO "%" + +%token <string_val> IDENTIFIER +%token <string_val> STRING +%token <string_val> FP_MAP_LOOKUP FP_ARRAY_LOOKUP +%token <double_val> FLOAT +%token <i64_val> INTEGER +%token <string_val> USER GROUP SCHEME NAMESPACE SPECIFIC BUCKET GID TYPE ORDER + +%type <string_val> ident mangled_ident +%type <abstract_node> bool_ + /* TODO 'leaf' is a bad name for something that isn't a leaf... */ +%type <abstract_node> expression comparison logical_expr leaf doc_type +%type <string_val> id_arg +%type <value_node> number null_ value string arith_expr id_spec variable +%type <field_expr_node> field_spec + +%destructor { delete $$; } IDENTIFIER STRING FP_MAP_LOOKUP FP_ARRAY_LOOKUP +%destructor { delete $$; } USER GROUP SCHEME NAMESPACE SPECIFIC BUCKET GID TYPE ORDER +%destructor { delete $$; } null_ bool_ number string doc_type ident id_arg id_spec +%destructor { delete $$; } variable mangled_ident field_spec value arith_expr +%destructor { delete $$; } comparison leaf logical_expr expression + +%start entry + +%parse-param {DocSelScanner& scanner} +%parse-param {const BucketIdFactory& bucket_id_factory} +%parse-param {const DocumentTypeRepo& doc_type_repo} +%parse-param {std::unique_ptr<Node>& recv_expr} + + /* Generated parser header file verbatim */ +%code requires { + +#include "location.hh" +#include <vespa/document/select/constant.h> +#include <vespa/document/select/branch.h> +#include <vespa/document/select/compare.h> +#include <vespa/document/select/valuenodes.h> +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace document { +class BucketIdFactory; +class DocumentTypeRepo; +} + +namespace document::select { +class DocSelScanner; +class Node; +class Constant; +class ValueNode; +} + +} + +%code { + +// Bison has some chunky destructors that trigger inlining warnings. Disable warning +// for this translation unit, since we can't really do much about the code it generates. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winline" + +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/select/scanner.h> +#include <vespa/document/select/constant.h> +#include <vespa/document/select/branch.h> +#include <vespa/document/select/compare.h> +#include <vespa/document/select/doctype.h> +#include <vespa/document/select/valuenodes.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/util/exceptions.h> +#include <string> +#include <iostream> +#include <sstream> +#include <memory> + +using string = vespalib::string; + +// Wrap grabbing pointers from sub-rules in a way that nulls out the +// stored attribute from the Bison stack. Otherwise, exception cleanup +// in the parser code will attempt to double-delete the pointee. +// Yes, it's not beautiful, but that's life when you're dealing with raw pointers. +template <typename T> +std::unique_ptr<T> steal(T*& ptr) noexcept { + std::unique_ptr<T> owned(ptr); + ptr = nullptr; + return owned; +} + +// yylex tokenization must defer to scanner instance given to parser +#undef yylex +#define yylex scanner.yylex + +} + +%code provides { + +// This cute little indirection is to get around the syntax_error constructor +// being defined as inline and therefore not being available outside the +// auto-generated parser source file. +[[noreturn]] void throw_parser_syntax_error(const document::select::DocSelParser::location_type& loc, + const std::string& msg); + +} + + +%left OR +%left AND +%left EQ NE LT GT LE GE GLOB REGEX +%left PLUS MINUS +%left MULTIPLY DIVIDE +%left MODULO /* Matches legacy parser recursive descent precedence */ +%precedence NEG +%right UNOT +%left NON_DOT +%precedence DOT /* Used to give higher precedence to id.foo vs id expressions. Re: "dangling else" problem */ + +%% + +null_ + : NULL { $$ = new NullValueNode(); } + ; + +bool_ + : TRUE { $$ = new Constant(true); } + | FALSE { $$ = new Constant(false); } + ; + +number + : INTEGER { $$ = new IntegerValueNode($1, false); } + | FLOAT { $$ = new FloatValueNode($1); } + ; + +string + : STRING { { + try { + $$ = new StringValueNode(StringUtil::unescape(*steal<string>($1))); + } catch (const vespalib::IllegalArgumentException& exc) { + throw syntax_error(@$, exc.getMessage()); + } + } } + ; + +doc_type + : ident { + if (doc_type_repo.getDocumentType(*$1) == nullptr) { + throw syntax_error(@$, vespalib::make_string("Document type '%s' not found", $1->c_str())); + } + $$ = new DocType(*steal<string>($1)); + } + ; + +ident + : IDENTIFIER { $$ = $1; } + | SCHEME { $$ = $1; } + | TYPE { $$ = $1; } + | NAMESPACE { $$ = $1; } + | SPECIFIC { $$ = $1; } + | BUCKET { $$ = $1; } + | GID { $$ = $1; } + | ORDER { $$ = $1; } + ; + +id_arg + : USER { $$ = $1; } + | GROUP { $$ = $1; } + | SCHEME { $$ = $1; } + | NAMESPACE { $$ = $1; } + | SPECIFIC { $$ = $1; } + | BUCKET { $$ = $1; } + | GID { $$ = $1; } + | TYPE { $$ = $1; } + ; + +id_spec + : ID %prec NON_DOT { $$ = new IdValueNode(bucket_id_factory, "id", ""); } /* Prefer shifting instead of reducing */ + | ID "." id_arg { $$ = new IdValueNode(bucket_id_factory, "id", *steal<string>($3)); } + | ID "." IDENTIFIER "(" ")" { $$ = new FunctionValueNode(*steal<string>($3), std::make_unique<IdValueNode>(bucket_id_factory, "id", "")); } + | ID "." ORDER "(" INTEGER "," INTEGER ")" { $$ = new IdValueNode(bucket_id_factory, "id", *steal<string>($3), $5, $7); } + ; + +variable + : "$" ident { $$ = new VariableValueNode(*steal<string>($2)); } + ; + + /* FIXME this is a horrible leftover of post-parsed fieldpath processing */ + /* At least we verify structural integrity at initial parse-time now... */ + /* Post-parsing should be replaced with an actual parse-time built AST! */ +mangled_ident + : ident { $$ = $1; } + | mangled_ident FP_MAP_LOOKUP { $1->append(*steal<string>($2)); $$ = $1; } + | mangled_ident FP_ARRAY_LOOKUP { $1->append(*steal<string>($2)); $$ = $1; } + ; + +field_spec + : ident "." mangled_ident { + if (doc_type_repo.getDocumentType(*$1) == nullptr) { + throw syntax_error(@$, vespalib::make_string("Document type '%s' not found", $1->c_str())); + } + $$ = new FieldExprNode(std::make_unique<FieldExprNode>(*steal<string>($1)), *steal<string>($3)); + } + | field_spec "." mangled_ident { $$ = new FieldExprNode(steal<FieldExprNode>($1), *steal<string>($3)); } + ; + +value + : null_ { $$ = $1; } + | string { $$ = $1; } + | id_spec { $$ = $1; } + | variable { $$ = $1; } + | NOW_FUNC { $$ = new CurrentTimeValueNode(); } + ; + +arith_expr + : value { $$ = $1; } + | number { $$ = $1; } + /* JavaCC and legacy parsers don't support unary plus/minus for _expressions_, just for numbers. So we have to fudge this a bit. */ + | "-" number %prec NEG { + if (dynamic_cast<IntegerValueNode*>($2) != nullptr) { + $$ = new IntegerValueNode(- static_cast<IntegerValueNode&>(*steal<ValueNode>($2)).getValue(), false); + } else { + $$ = new FloatValueNode(- dynamic_cast<FloatValueNode&>(*steal<ValueNode>($2)).getValue()); + } + } + | "+" number %prec NEG { $$ = $2; } + | field_spec { $$ = steal<FieldExprNode>($1)->convert_to_field_value().release(); } + | field_spec "(" ")" { $$ = steal<FieldExprNode>($1)->convert_to_function_call().release(); } + | arith_expr "+" arith_expr { $$ = new ArithmeticValueNode(steal<ValueNode>($1), "+", steal<ValueNode>($3)); } + | arith_expr "-" arith_expr { $$ = new ArithmeticValueNode(steal<ValueNode>($1), "-", steal<ValueNode>($3)); } + | arith_expr "*" arith_expr { $$ = new ArithmeticValueNode(steal<ValueNode>($1), "*", steal<ValueNode>($3)); } + | arith_expr "/" arith_expr { $$ = new ArithmeticValueNode(steal<ValueNode>($1), "/", steal<ValueNode>($3)); } + | arith_expr "%" arith_expr { $$ = new ArithmeticValueNode(steal<ValueNode>($1), "%", steal<ValueNode>($3)); } + | "(" arith_expr ")" { $$ = $2; $$->setParentheses(); } + | arith_expr "." IDENTIFIER "(" ")" { $$ = new FunctionValueNode(*steal<string>($3), steal<ValueNode>($1)); } /* FIXME shift/reduce conflict */ + ; + +comparison + : arith_expr EQ arith_expr { $$ = new Compare(steal<ValueNode>($1), FunctionOperator::EQ, steal<ValueNode>($3), bucket_id_factory); } + | arith_expr NE arith_expr { $$ = new Compare(steal<ValueNode>($1), FunctionOperator::NE, steal<ValueNode>($3), bucket_id_factory); } + | arith_expr GE arith_expr { $$ = new Compare(steal<ValueNode>($1), FunctionOperator::GEQ, steal<ValueNode>($3), bucket_id_factory); } + | arith_expr LE arith_expr { $$ = new Compare(steal<ValueNode>($1), FunctionOperator::LEQ, steal<ValueNode>($3), bucket_id_factory); } + | arith_expr GT arith_expr { $$ = new Compare(steal<ValueNode>($1), FunctionOperator::GT, steal<ValueNode>($3), bucket_id_factory); } + | arith_expr LT arith_expr { $$ = new Compare(steal<ValueNode>($1), FunctionOperator::LT, steal<ValueNode>($3), bucket_id_factory); } + | arith_expr GLOB arith_expr { $$ = new Compare(steal<ValueNode>($1), GlobOperator::GLOB, steal<ValueNode>($3), bucket_id_factory); } + | arith_expr REGEX arith_expr { $$ = new Compare(steal<ValueNode>($1), RegexOperator::REGEX, steal<ValueNode>($3), bucket_id_factory); } + ; + +leaf + : bool_ { $$ = $1; } + | comparison { $$ = $1; } + | doc_type { $$ = $1; } + | arith_expr { /* Actually field_spec, see comment below..! */ + // Grammar-wise, we _do not_ accept arbitrary arith_exprs at this level. But the + // selection grammar as it stands is otherwise ambiguous with LR(1) parsing. + // More specifically, if we used field_spec instead of arith_expr here, the parser + // state machine cannot decide what to do if it has processed the sequence '(' field_spec + // and sees the next token of ')'. Since both logical_expr and arith_expr allows for + // parenthesis expression recursion, the reduce step may produce either of these and + // is therefore technically undefined. By using arith_expr instead for this rule, all + // '(' field_spec ')' sequences result in an arith_expr rule match and the reduce/reduce + // conflict goes away. We can then do a sneaky "run-time" type check to ensure we only + // get the expected node from the rule. + // It's not pretty, but it avoids an undefined grammar (which is much less pretty!). + auto node = steal<ValueNode>($1); + if (dynamic_cast<FieldValueNode*>(node.get()) == nullptr) { + throw syntax_error(@$, "expected field spec, doctype, bool or comparison"); + } + // Implicit rewrite to non-null comparison node + $$ = new Compare(std::move(node), + FunctionOperator::NE, + std::make_unique<NullValueNode>(), + bucket_id_factory); + } + ; + +logical_expr + : leaf { $$ = $1; } + | logical_expr AND logical_expr { $$ = new And(steal<Node>($1), steal<Node>($3)); } + | logical_expr OR logical_expr { $$ = new Or(steal<Node>($1), steal<Node>($3)); } + | NOT logical_expr %prec UNOT { $$ = new Not(steal<Node>($2)); } + | "(" logical_expr ")" { $$ = $2; $$->setParentheses(); } + ; + +expression + : logical_expr { $$ = $1; } + ; + +entry + : expression END { recv_expr = steal<Node>($1); } + | END { recv_expr = std::make_unique<Constant>(true); } + ; + +%% + +void document::select::DocSelParser::error(const location_type& l, const std::string& what) { + throw syntax_error(l, what); +} + +void throw_parser_syntax_error(const document::select::DocSelParser::location_type& loc, const std::string& msg) { + throw document::select::DocSelParser::syntax_error(loc, msg); +} + +#pragma GCC diagnostic pop diff --git a/document/src/vespa/document/select/node.h b/document/src/vespa/document/select/node.h index eab46e824c8..83e2ea3542d 100644 --- a/document/src/vespa/document/select/node.h +++ b/document/src/vespa/document/select/node.h @@ -5,9 +5,8 @@ * * @brief Base class for all nodes in the document selection tree. * - * @author H�kon Humberset + * @author Håkon Humberset * @date 2005-06-07 - * @version $Id$ */ #pragma once diff --git a/document/src/vespa/document/select/orderingselector.cpp b/document/src/vespa/document/select/orderingselector.cpp index bf8e96c0533..836647aab26 100644 --- a/document/src/vespa/document/select/orderingselector.cpp +++ b/document/src/vespa/document/select/orderingselector.cpp @@ -137,7 +137,6 @@ namespace { void visitArithmeticValueNode(const ArithmeticValueNode &) override {} void visitFunctionValueNode(const FunctionValueNode &) override {} void visitIdValueNode(const IdValueNode &) override {} - void visitSearchColumnValueNode(const SearchColumnValueNode &) override {} void visitFieldValueNode(const FieldValueNode &) override {} void visitFloatValueNode(const FloatValueNode &) override {} void visitVariableValueNode(const VariableValueNode &) override {} diff --git a/document/src/vespa/document/select/parse_utils.cpp b/document/src/vespa/document/select/parse_utils.cpp new file mode 100644 index 00000000000..ab4ce2f6d4a --- /dev/null +++ b/document/src/vespa/document/select/parse_utils.cpp @@ -0,0 +1,37 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "parse_utils.h" +#include <boost/spirit/include/qi.hpp> + +namespace document::select::util { + +namespace qi = boost::spirit::qi; + +// TODO replace use of Spirit.Qi with std::from_string when available. +// Note: these parsers are all pure, reentrant and without locking. +bool parse_hex_i64(const char* str, size_t len, int64_t& out) { + const char* iter = str; + const char* end = str + len; + // Legacy parser parses hex numbers as u64 rather than i64 (then implicitly + // converts), so we do the same thing here to avoid change of semantics. + using u64_hex_parser = qi::uint_parser<uint64_t, 16, 1, 16>; + u64_hex_parser u64_hex; + uint64_t tmp = 0; + const bool ok = qi::parse(iter, end, u64_hex, tmp); + out = static_cast<int64_t>(tmp); + return (ok && (iter == end)); +} +bool parse_i64(const char* str, size_t len, int64_t& out) { + const char* iter = str; + const char* end = str + len; + const bool ok = qi::parse(iter, end, qi::long_long, out); + return (ok && (iter == end)); +} +bool parse_double(const char* str, size_t len, double& out) { + const char* iter = str; + const char* end = str + len; + const bool ok = qi::parse(iter, end, qi::double_, out); + return (ok && (iter == end)); +} + +} diff --git a/document/src/vespa/document/select/parse_utils.h b/document/src/vespa/document/select/parse_utils.h new file mode 100644 index 00000000000..38c36dfe94a --- /dev/null +++ b/document/src/vespa/document/select/parse_utils.h @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <stdint.h> +#include <stddef.h> + +namespace document::select::util { + +// Fast, locale-independent numeric parse helpers for Flex lexing. + +// For all parse_* functions, returns true if parsing is successful. False otherwise. +// Value of `out` is undefined if return value is false. +bool parse_hex_i64(const char* str, size_t len, int64_t& out); +bool parse_i64(const char* str, size_t len, int64_t& out); +bool parse_double(const char* str, size_t len, double& out); + +}
\ No newline at end of file diff --git a/document/src/vespa/document/select/parser.cpp b/document/src/vespa/document/select/parser.cpp index ceaf0b0c438..9f015409011 100644 --- a/document/src/vespa/document/select/parser.cpp +++ b/document/src/vespa/document/select/parser.cpp @@ -1,1493 +1,33 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include "parser.h" -#include "branch.h" -#include "compare.h" -#include "constant.h" -#include "operator.h" -#include "doctype.h" -#include "valuenode.h" -#include "simpleparser.h" - -#include <vespa/document/repo/documenttyperepo.h> +#include "scanner.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/util/stringutil.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/locale/c.h> -#include <boost/spirit/include/classic_chset.hpp> -#include <boost/spirit/include/classic_core.hpp> -#include <boost/spirit/include/classic_escape_char.hpp> -#include <boost/spirit/include/classic_grammar_def.hpp> -#include <boost/spirit/include/classic_parse_tree.hpp> -#include <boost/spirit/include/classic_tree_to_xml.hpp> -#include <iostream> -#include <map> +#include <vespa/vespalib/util/stringfmt.h> #include <sstream> -using boost::spirit::classic::tree_node; -using document::DocumentTypeRepo; -using std::unique_ptr; -using std::cerr; -using std::endl; -using std::istringstream; -using std::ostringstream; -using vespalib::IllegalStateException; - -/* - * This cannot be part of a plugin. boost contains constructs causing - * compiler to generate calls to atexit(). - */ - -#define parse_assert(a) - -namespace document { -namespace select { - -VESPA_IMPLEMENT_EXCEPTION(ParsingFailedException, vespalib::Exception); - -Parser::Parser(const DocumentTypeRepo& repo, - const BucketIdFactory& bucketIdFactory) - : _repo(repo), - _bucketIdFactory(bucketIdFactory) -{ -} - -namespace { - -/** - * Defines the grammar for the document selection text format. - */ -struct DocSelectionGrammar - : public boost::spirit::classic::grammar<DocSelectionGrammar> -{ - /** Node identifiers (value 0 should not be used) */ - enum ids { id_nil=1, id_bool, id_number, id_string, - id_doctype, id_fieldname, id_function, id_idarg, id_searchcolumnarg, - id_operator, id_idspec, id_searchcolumnspec, id_fieldspec, id_value, - id_valuefuncadd, id_valuefuncmul, id_valuefuncmod, - id_valuegroup, id_arithmvalue, - id_comparison, id_leaf, id_not, id_and, - id_or, id_group, id_order, id_expression, id_variable }; - - const DocumentTypeRepo &_repo; - const BucketIdFactory& _bucketIdFactory; - - DocSelectionGrammar(const DocumentTypeRepo& repo, - const BucketIdFactory& bucketIdFactory) - : _repo(repo), - _bucketIdFactory(bucketIdFactory) {} - - const BucketIdFactory& getBucketIdFactory() const - { return _bucketIdFactory; } - - /** Grammar base types. To be able to retrieve different grammars. */ - template <typename Scanner> - struct gram_base { - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_nil> > rule_nil; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_bool> > rule_bool; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_number> > rule_number; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_string> > rule_string; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_doctype> > rule_doctype; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_fieldname> > rule_fieldname; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_function> > rule_function; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_idarg> > rule_idarg; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_searchcolumnarg> > rule_searchcolumnarg; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_operator> > rule_operator; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_idspec> > rule_idspec; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_searchcolumnspec> > rule_searchcolumnspec; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_fieldspec> > rule_fieldspec; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_value> > rule_value; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_valuefuncadd> > rule_valuefuncadd; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_valuefuncmul> > rule_valuefuncmul; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_valuefuncmod> > rule_valuefuncmod; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_valuegroup> > rule_valuegroup; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_arithmvalue> > rule_arithmvalue; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_comparison> > rule_comparison; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_leaf> > rule_leaf; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_not> > rule_not; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_and> > rule_and; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_or> > rule_or; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_group> > rule_group; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_order> > rule_order; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_expression> > rule_expression; - typedef typename boost::spirit::classic::rule<Scanner, - boost::spirit::classic::parser_tag<id_variable> > rule_variable; - typedef boost::spirit::classic::grammar_def<rule_expression, - rule_leaf, - rule_arithmvalue> type; - }; - - template <typename Scanner> - struct definition : gram_base<Scanner>::type - { - typename gram_base<Scanner>::rule_nil _nil; - typename gram_base<Scanner>::rule_bool _bool; - typename gram_base<Scanner>::rule_number _number; - typename gram_base<Scanner>::rule_string _string; - typename gram_base<Scanner>::rule_doctype _doctype; - typename gram_base<Scanner>::rule_fieldname _fieldname; - typename gram_base<Scanner>::rule_function _function; - typename gram_base<Scanner>::rule_idarg _idarg; - typename gram_base<Scanner>::rule_searchcolumnarg _searchcolumnarg; - typename gram_base<Scanner>::rule_operator _operator; - typename gram_base<Scanner>::rule_idspec _idspec; - typename gram_base<Scanner>::rule_searchcolumnspec _searchcolumnspec; - typename gram_base<Scanner>::rule_fieldspec _fieldspec; - typename gram_base<Scanner>::rule_value _value; - typename gram_base<Scanner>::rule_valuefuncadd _valuefuncadd; - typename gram_base<Scanner>::rule_valuefuncmul _valuefuncmul; - typename gram_base<Scanner>::rule_valuefuncmod _valuefuncmod; - typename gram_base<Scanner>::rule_valuegroup _valuegroup; - typename gram_base<Scanner>::rule_arithmvalue _arithmvalue; - typename gram_base<Scanner>::rule_comparison _comparison; - typename gram_base<Scanner>::rule_leaf _leaf; - typename gram_base<Scanner>::rule_not _not; - typename gram_base<Scanner>::rule_and _and; - typename gram_base<Scanner>::rule_or _or; - typename gram_base<Scanner>::rule_group _group; - typename gram_base<Scanner>::rule_order _order; - typename gram_base<Scanner>::rule_expression _expression; - typename gram_base<Scanner>::rule_variable _variable; +namespace document::select { - definition(const DocSelectionGrammar&) - : _nil(), - _bool(), - _number(), - _string(), - _doctype(), - _fieldname(), - _function(), - _idarg(), - _operator(), - _idspec(), - _searchcolumnspec(), - _fieldspec(), - _value(), - _valuefuncadd(), - _valuefuncmul(), - _valuefuncmod(), - _valuegroup(), - _arithmvalue(), - _comparison(), - _leaf(), - _not(), - _and(), - _or(), - _group(), - _order(), - _expression(), - _variable() - { - using namespace boost::spirit::classic; +std::unique_ptr<Node> Parser::parse(const std::string& str) const { + try { + std::istringstream ss(str); + DocSelScanner scanner(&ss); - boost::spirit::classic::uint_parser<uint64_t, 16, 1, -1> hexvalue; - - // Initialize primitives - _nil = lexeme_d[ as_lower_d["null"] ]; - _bool = lexeme_d[ as_lower_d["true"] | as_lower_d["false"] ]; - _number = lexeme_d[ str_p("0x") >> hexvalue ] | lexeme_d[ real_p ]; - _string = ( lexeme_d[ - ( no_node_d[ ch_p('"') ] >> - token_node_d[ *( ~chset<>("\\\"\x00-\x1f\x7f-\xff") | - ( '\\' >> ( ch_p('\\') | 't' | 'n' | 'f' | 'r' | '"' | - (ch_p('x') >> xdigit_p >> xdigit_p) ) ) ) ] >> - no_node_d[ ch_p('"') ] ) | - ( no_node_d[ ch_p('\'') ] >> - token_node_d[ *( ~chset<>("\\'\x00-\x1f\x7f-\xff") | - ( '\\' >> ( ch_p('\\') | 't' | 'n' | 'f' | 'r' | '\'' | - (ch_p('x') >> xdigit_p >> xdigit_p) ) ) ) ] >> - no_node_d[ ch_p('\'') ] ) - ] ); - _doctype = lexeme_d[ token_node_d[ chset<>("_A-Za-z") - >> *(chset<>("_A-Za-z0-9")) ]]; - _fieldname = lexeme_d[ token_node_d[chset<>("_A-Za-z") - >> *(chset<>("_A-Za-z0-9{}[]$")) - ]]; - _function = lexeme_d[ token_node_d[ chset<>("A-Za-z") - >> *(chset<>("A-Za-z0-9")) ] - >> no_node_d[ str_p("()") ] ]; - - _order = as_lower_d["order"] - >> no_node_d[ ch_p('(') ] - >> _number - >> no_node_d[ ch_p(',') ] - >> _number - >> no_node_d[ ch_p(')') ]; - - _idarg = (as_lower_d[ "scheme"] | as_lower_d[ "namespace"] | - as_lower_d[ "specific" ] | as_lower_d[ "user" ] | - as_lower_d[ "group" ] | as_lower_d[ "bucket" ] | - as_lower_d[ "gid" ] | as_lower_d["type"] | _order); - - _searchcolumnarg = lexeme_d[ token_node_d[ *(chset<>("_A-Za-z0-9")) ]]; - _operator = (str_p(">=") | ">" | "==" | "=~" | "=" - | "<=" | "<" | "!="); - // Derived - _idspec = as_lower_d["id"] - >> !(no_node_d[ ch_p('.') ] >> _idarg); - _searchcolumnspec = as_lower_d["searchcolumn"] - >> !(no_node_d[ ch_p('.') ] >> _searchcolumnarg); - _fieldspec = _doctype - >> +( no_node_d[ ch_p('.') ] >> (_function | _fieldname)); - _variable = lexeme_d[ token_node_d[chset<>("$") - >> *(chset<>("A-Za-z0-9")) - ]]; - _value = (_valuegroup | _function | _nil | _number | _string - | _idspec | _searchcolumnspec | _fieldspec | _variable) - >> *(no_node_d[ ch_p('.') ] >> _function); - _valuefuncmod = (_valuegroup | _value) - >> +( ch_p('%') - >> (_valuegroup | _value) ); - _valuefuncmul = (_valuefuncmod | _valuegroup | _value) - >> +( (ch_p('*') | ch_p('/')) - >> (_valuefuncmod | _valuegroup | _value)); - _valuefuncadd - = (_valuefuncmul | _valuefuncmod | _valuegroup | _value) - >> +((ch_p('+') | ch_p('-')) - >> (_valuefuncmul | _valuefuncmod | _valuegroup | - _value)); - _valuegroup = no_node_d[ ch_p('(') ] >> _arithmvalue - >> no_node_d[ ch_p(')') ] - >> *(no_node_d[ ch_p('.') ] >> _function); - _arithmvalue = (_valuefuncadd | _valuefuncmul | _valuefuncmod - | _valuegroup | _value); - _comparison = _arithmvalue >> _operator >> _arithmvalue; - _leaf = _bool | _comparison | _fieldspec | _doctype; - - _not = (as_lower_d["not"] >> _group) - | (lexeme_d[ as_lower_d["not"] >> no_node_d[ space_p ] ] >> _leaf); - _and = (_not | _group | _leaf) - >> as_lower_d["and"] >> (_and | _not | _group | _leaf); - _or = (_and | _not | _group | _leaf) - >> as_lower_d["or"] >> (_or | _and | _not | _group | _leaf); - _group = no_node_d[ ch_p('(') ] - >> (_or | _and | _not | _group | _leaf) - >> no_node_d[ ch_p(')') ]; - - _expression = !(_or | _and | _not | _group | _leaf) >> end_p; - - this->start_parsers(_expression, _leaf, _arithmvalue); + std::unique_ptr<Node> root; + DocSelParser parser(scanner, _bucket_id_factory, _doc_type_repo, root); + if (parser.parse() != 0) { + throw ParsingFailedException( + vespalib::make_string("Unknown parse failure while parsing selection '%s'", str.c_str()), + VESPA_STRLOC); } - }; - -}; - -template<typename T> -std::unique_ptr<Node> -parseTree(DocSelectionGrammar& grammar, tree_node<T>& root) { - return parseNode(grammar, root); -} - -template<typename T> -std::unique_ptr<Node> -parseNode(DocSelectionGrammar& grammar, tree_node<T>& node) { - switch (node.value.id().to_long()) { - case DocSelectionGrammar::id_or: - return parseOr(grammar, node); - case DocSelectionGrammar::id_and: - return parseAnd(grammar, node); - case DocSelectionGrammar::id_not: - return parseNot(grammar, node); - case DocSelectionGrammar::id_group: - { - std::unique_ptr<Node> n(parseNode(grammar, node.children[0])); - n->setParentheses(); - return n; - } - case DocSelectionGrammar::id_leaf: - case DocSelectionGrammar::id_value: - parse_assert(node.children.size() == 1); - return parseNode(grammar, node.children[0]); - case DocSelectionGrammar::id_expression: - if (node.children.size() == 1) { - return parseNode(grammar, node.children[0]); - } - parse_assert(node.children.size() == 0); - return std::unique_ptr<Node>(new Constant("true")); - case DocSelectionGrammar::id_bool: - return parseBool(grammar, node); - case DocSelectionGrammar::id_comparison: - return parseComparison(grammar, node); - case DocSelectionGrammar::id_fieldspec: - return parseFieldSpec(grammar, node); - case DocSelectionGrammar::id_doctype: - return parseDocType(grammar, node); - } - vespalib::asciistream ost; - ost << "Received unhandled nodetype " - << node.value.id().to_long() << " in parseNode()\n"; - throw IllegalStateException(ost.str(), VESPA_STRLOC); -} - -template<typename T> -std::unique_ptr<Node> -parseOr(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_or); - parse_assert(node.children.size() == 3); - vespalib::string op(node.children[1].value.begin(), - node.children[1].value.end()); - return std::unique_ptr<Node>(new Or( - parseNode(grammar, node.children[0]), - parseNode(grammar, node.children[2]), - op.c_str())); -} - -template<typename T> -std::unique_ptr<Node> -parseAnd(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_and); - parse_assert(node.children.size() == 3); - vespalib::string op(node.children[1].value.begin(), - node.children[1].value.end()); - return std::unique_ptr<Node>(new And( - parseNode(grammar, node.children[0]), - parseNode(grammar, node.children[2]), - op.c_str())); -} - -template<typename T> -std::unique_ptr<Node> -parseNot(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_not); - parse_assert(node.children.size() == 2); - vespalib::string op(node.children[0].value.begin(), - node.children[0].value.end()); - return std::unique_ptr<Node>(new Not( - parseNode(grammar, node.children[1]), op.c_str())); -} - -template<typename T> -std::unique_ptr<Node> -parseBool(DocSelectionGrammar& grammar, tree_node<T>& node) { - (void) grammar; - parse_assert(node.value.id().to_long() == grammar.id_bool); - parse_assert(node.children.size() == 1); - parse_assert(node.children[0].value.id().to_long() == grammar.id_bool); - parse_assert(node.children[0].children.size() == 0); - vespalib::string s(node.children[0].value.begin(), node.children[0].value.end()); - return std::unique_ptr<Node>(new Constant(s)); -} - -template<typename T> -std::unique_ptr<Node> -parseComparison(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_comparison); - parse_assert(node.children.size() == 3); - parse_assert(node.children[1].children.size() == 1); - vespalib::string op(node.children[1].children[0].value.begin(), - node.children[1].children[0].value.end()); - return std::unique_ptr<Node>(new Compare( - parseArithmValue(grammar, node.children[0]), - Operator::get(op), - parseArithmValue(grammar, node.children[2]), - grammar.getBucketIdFactory())); -} - -template<typename T> -std::unique_ptr<Node> -parseFieldSpec(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_fieldspec); - return std::unique_ptr<Node>(new Compare( - parseFieldSpecValue(grammar, node), - Operator::get("!="), - std::unique_ptr<ValueNode>(new NullValueNode("null")), - grammar.getBucketIdFactory())); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseVariable(DocSelectionGrammar& grammar, tree_node<T>& node) { - (void) grammar; - parse_assert(node.value.id().to_long() == grammar.id_variable); - vespalib::string varName(node.children[0].value.begin(), - node.children[0].value.end()); - return std::unique_ptr<ValueNode>(new VariableValueNode(varName.substr(1))); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseGlobValueFunction(DocSelectionGrammar& grammar, tree_node<T>& node) { - (void) grammar; - parse_assert(node.value.id().to_long() == grammar.id_function); - vespalib::string varName(node.children[0].value.begin(), - node.children[0].value.end()); - if (varName == "now") { - return std::unique_ptr<ValueNode>(new CurrentTimeValueNode); + return root; + } catch (const DocSelParser::syntax_error& err) { + throw ParsingFailedException( + vespalib::make_string("%s at column %u when parsing selection '%s'", + err.what(), err.location.begin.column, str.c_str()), + VESPA_STRLOC); } - throw ParsingFailedException("Unexpected function name '" + varName - + "' found.", VESPA_STRLOC); } -template<typename T> -std::unique_ptr<Node> -parseDocType(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_doctype); - parse_assert(node.children.size() == 1); - parse_assert(node.children[0].value.id().to_long() == grammar.id_doctype); - parse_assert(node.children[0].children.size() == 0); - vespalib::string doctype(node.children[0].value.begin(), - node.children[0].value.end()); - // Verify existance of any version of document - if (!grammar._repo.getDocumentType(doctype)) { - throw ParsingFailedException("Document type " + doctype + " not found", - VESPA_STRLOC); - } - return std::unique_ptr<Node>(new DocType(doctype)); -} - -template<typename T> -std::unique_ptr<ValueNode> -addFunctions(DocSelectionGrammar& grammar, tree_node<T>& node, - std::unique_ptr<ValueNode> src, uint32_t index) -{ - (void) grammar; - while (index < node.children.size()) { - parse_assert(node.children[index].value.id().to_long() - == grammar.id_function); - vespalib::string func(node.children[index].children[0].value.begin(), - node.children[index].children[0].value.end()); - std::unique_ptr<ValueNode> fnode(new FunctionValueNode(func, std::move(src))); - src = std::move(fnode); - ++index; - } - return std::move(src); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseArithmValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - switch (node.value.id().to_long()) { - case DocSelectionGrammar::id_arithmvalue: - parse_assert(node.children.size() == 1); - return parseArithmValue(grammar, node.children[0]); - case DocSelectionGrammar::id_value: - return parseValue(grammar, node); - case DocSelectionGrammar::id_valuegroup: - return parseValueGroup(grammar, node); - case DocSelectionGrammar::id_valuefuncadd: - case DocSelectionGrammar::id_valuefuncmul: - case DocSelectionGrammar::id_valuefuncmod: - return parseValueArithmetics(grammar, node); - } - vespalib::asciistream ost; - ost << "Received unhandled nodetype " - << node.value.id().to_long() - << " in parseArithmValue()\n"; - throw IllegalStateException(ost.str(), VESPA_STRLOC); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseValueArithmetics(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.children.size() >= 3 && node.children.size() % 2 == 1); - std::unique_ptr<ValueNode> lhs(parseArithmValue(grammar, node.children[0])); - for (unsigned int i = 1; i < node.children.size(); i += 2) { - vespalib::string op(node.children[i].value.begin(), - node.children[i].value.end()); - std::unique_ptr<ValueNode> rhs(parseArithmValue(grammar, - node.children[i + 1])); - std::unique_ptr<ValueNode> res( - new ArithmeticValueNode(std::move(lhs), op, std::move(rhs))); - lhs = std::move(res); - } - return lhs; -} - -template<typename T> -std::unique_ptr<ValueNode> -parseValueGroup(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_valuegroup); - parse_assert(node.children.size() >= 1); - std::unique_ptr<ValueNode> result( - parseArithmValue(grammar, node.children[0])); - result->setParentheses(); - return addFunctions(grammar, node, std::move(result), 1); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_value); - parse_assert(node.children.size() >= 1); - std::unique_ptr<ValueNode> result; - switch (node.children[0].value.id().to_long()) { - case DocSelectionGrammar::id_nil: - result = parseNilValue(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_idspec: - result = parseIdSpecValue(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_searchcolumnspec: - result = parseSearchColumnSpecValue(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_fieldspec: - result = parseFieldSpecValue(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_number: - result = parseNumberValue(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_string: - result = parseStringValue(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_valuegroup: - result = parseValueGroup(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_variable: - result = parseVariable(grammar, node.children[0]); - break; - case DocSelectionGrammar::id_function: - result = parseGlobValueFunction(grammar, node.children[0]); - break; - default: - vespalib::asciistream ost; - ost << "Received unhandled nodetype " - << node.children[0].value.id().to_long() - << " in parseValue(), from node of type " - << node.value.id().to_long() << "\n"; - throw IllegalStateException(ost.str(), VESPA_STRLOC); - } - return addFunctions(grammar, node, std::move(result), 1); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseNilValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - (void) grammar; - parse_assert(node.value.id().to_long() == grammar.id_nil); - parse_assert(node.children.size() == 1); - parse_assert(node.children[0].children.size() == 0); - vespalib::string op(node.children[0].value.begin(), - node.children[0].value.end()); - return std::unique_ptr<ValueNode>(new NullValueNode(op)); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseIdSpecValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_idspec); - parse_assert(node.children.size() >= 1); - parse_assert(node.children[0].children.size() == 0); - vespalib::string id(node.children[0].value.begin(), - node.children[0].value.end()); - if (node.children.size() == 1) { - return std::unique_ptr<ValueNode>( - new IdValueNode(grammar.getBucketIdFactory(), id, "")); - } - - vespalib::string type; - - int widthBits = -1; - int divisionBits = -1; - - if (node.children[1].children[0].value.id().to_long() == grammar.id_order) { - tree_node<T>& ordernode(node.children[1].children[0]); - type = vespalib::string(ordernode.children[0].value.begin(), - ordernode.children[0].value.end()); - - vespalib::string val = vespalib::string( - ordernode.children[1].children[0].value.begin(), - ordernode.children[1].children[0].value.end()); - widthBits = atoi(val.c_str()); - - val = vespalib::string(ordernode.children[2].children[0].value.begin(), - ordernode.children[2].children[0].value.end()); - divisionBits = atoi(val.c_str()); - } else { - type = vespalib::string(node.children[1].children[0].value.begin(), - node.children[1].children[0].value.end()); - } - - return std::unique_ptr<ValueNode>( - new IdValueNode(grammar.getBucketIdFactory(), id, type, - widthBits, divisionBits)); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseSearchColumnSpecValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_searchcolumnspec); - parse_assert(node.children.size() == 2); - parse_assert(node.children[0].children.size() == 0); - parse_assert(node.children[1].value.id().to_long() == grammar.id_searchcolumnarg); - - vespalib::string id(node.children[0].value.begin(), - node.children[0].value.end()); - parse_assert(node.children.size() == 2); - - vespalib::string val = vespalib::string(node.children[1].children[0].value.begin(), - node.children[1].children[0].value.end()); - return std::unique_ptr<ValueNode>(new SearchColumnValueNode( - grammar.getBucketIdFactory(), id, atoi(val.c_str()))); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseFieldSpecValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - parse_assert(node.value.id().to_long() == grammar.id_fieldspec); - parse_assert(node.children.size() >= 2); - parse_assert(node.children[0].value.id().to_long() == grammar.id_doctype); - vespalib::string doctype(node.children[0].children[0].value.begin(), - node.children[0].children[0].value.end()); - // Verify that document type exist at any version - if (!grammar._repo.getDocumentType(doctype)) { - throw ParsingFailedException("Document type " + doctype + " not found", - VESPA_STRLOC); - } - std::unique_ptr<ValueNode> value; - uint32_t iterator = 2; - - parse_assert(node.children[1].value.id().to_long() == grammar.id_fieldname); - vespalib::string field(node.children[1].children[0].value.begin(), - node.children[1].children[0].value.end()); - while (iterator < node.children.size() - && node.children[iterator].value.id().to_long() == grammar.id_fieldname) - { - field += "." + vespalib::string( - node.children[iterator].children[0].value.begin(), - node.children[iterator].children[0].value.end()); - ++iterator; - } - value.reset(new FieldValueNode(doctype, field)); - - for (; iterator<node.children.size(); ++iterator) { - std::unique_ptr<ValueNode> child(std::move(value)); - vespalib::string function(node.children[iterator].children[0].value.begin(), - node.children[iterator].children[0].value.end()); - parse_assert(node.children[iterator].value.id().to_long() == grammar.id_function); - value.reset(new FunctionValueNode(function, std::move(child))); - } - return value; -} - -template<typename T> -std::unique_ptr<ValueNode> -parseNumberValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - (void) grammar; - parse_assert(node.value.id().to_long() == grammar.id_number); - vespalib::string sval; - int base = 10; - if (node.children.size() == 2) { - base = 16; - sval = vespalib::string(node.children[1].value.begin(), - node.children[1].value.end()); - parse_assert(node.children[0].value.id().to_long() == grammar.id_number); - parse_assert(node.children[1].value.id().to_long() == grammar.id_number); - } else { - parse_assert(node.children.size() == 1); - sval = vespalib::string(node.children[0].value.begin(), - node.children[0].value.end()); - parse_assert(node.children[0].value.id().to_long() == grammar.id_number); - } - if (sval.find('.') != vespalib::string::npos) { - char* endptr; - double val = vespalib::locale::c::strtod(sval.c_str(), &endptr); - if (*endptr == '\0') { - return std::unique_ptr<ValueNode>(new FloatValueNode(val)); - } - } else { - char* endptr; - int64_t val; - if (base == 16) { - val = strtoull(sval.c_str(), &endptr, base); - } else { - val = strtoll(sval.c_str(), &endptr, base); - } - if (*endptr == '\0') { - return std::unique_ptr<ValueNode>(new IntegerValueNode(val, false)); - } - } - vespalib::string error = "'" + sval + "' is not a valid number."; - throw ParsingFailedException(error, VESPA_STRLOC); -} - -template<typename T> -std::unique_ptr<ValueNode> -parseStringValue(DocSelectionGrammar& grammar, tree_node<T>& node) { - (void) grammar; - parse_assert(node.value.id().to_long() == grammar.id_string); - if (node.children.size() == 0) { - return std::unique_ptr<ValueNode>(new StringValueNode("")); - } - parse_assert(node.children.size() == 1); - parse_assert(node.children[0].value.id().to_long() == grammar.id_string); - vespalib::string val(node.children[0].value.begin(), - node.children[0].value.end()); - return std::unique_ptr<ValueNode>(new StringValueNode(StringUtil::unescape(val))); -} - -template<typename Tree> -void printSpiritTree(std::ostream& out, Tree tree, const vespalib::string& query, - const DocSelectionGrammar& grammar) { - using boost::spirit::classic::parser_id; - - std::map<parser_id, vespalib::string> names; - names[parser_id(grammar.id_bool)] = "bool"; - names[parser_id(grammar.id_number)] = "number"; - names[parser_id(grammar.id_string)] = "string"; - names[parser_id(grammar.id_doctype)] = "doctype"; - names[parser_id(grammar.id_fieldname)] = "fieldname"; - names[parser_id(grammar.id_function)] = "function"; - names[parser_id(grammar.id_idarg)] = "idarg"; - names[parser_id(grammar.id_searchcolumnarg)] = "searchcolumnarg"; - names[parser_id(grammar.id_operator)] = "operator"; - names[parser_id(grammar.id_idspec)] = "idspec"; - names[parser_id(grammar.id_searchcolumnspec)] = "searchcolumnspec"; - names[parser_id(grammar.id_fieldspec)] = "fieldspec"; - names[parser_id(grammar.id_value)] = "value"; - names[parser_id(grammar.id_valuefuncadd)] = "valuefuncadd"; - names[parser_id(grammar.id_valuefuncmul)] = "valuefuncmul"; - names[parser_id(grammar.id_valuefuncmod)] = "valuefuncmod"; - names[parser_id(grammar.id_valuegroup)] = "valuegroup"; - names[parser_id(grammar.id_arithmvalue)] = "arithmvalue"; - names[parser_id(grammar.id_comparison)] = "comparison"; - names[parser_id(grammar.id_leaf)] = "leaf"; - names[parser_id(grammar.id_not)] = "not"; - names[parser_id(grammar.id_and)] = "and"; - names[parser_id(grammar.id_or)] = "or"; - names[parser_id(grammar.id_group)] = "group"; - names[parser_id(grammar.id_expression)] = "expression"; - tree_to_xml(out, tree, query.c_str(), names); -} - -template<typename Parser> -bool testExpr(const DocumentTypeRepo& repo, - const BucketIdFactory& factory, - const vespalib::string& expression, const Parser& parser, - const vespalib::string& result) -{ - //std::cerr << "Testing expression '" << expression << "'.\n"; - using boost::spirit::classic::space_p; - - DocSelectionGrammar grammar(repo, factory); - boost::spirit::classic::tree_parse_info<> info; - info = pt_parse(expression.c_str(), parser, - space_p); - std::ostringstream ost; - printSpiritTree(ost, info.trees, expression, grammar); - if (!info.full) { - cerr << "Expression '" << expression - << "' wasn't completely parsed\n" - << ost.str() << "\n"; - return false; - } - vespalib::string httpexpr = expression; - vespalib::string::size_type index; - while ((index = httpexpr.find('>')) != vespalib::string::npos) { - httpexpr = httpexpr.substr(0,index) + ">" - + httpexpr.substr(index+1); - } - vespalib::string fullresult = "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>\n" - "<!DOCTYPE parsetree SYSTEM \"parsetree.dtd\">\n" - "<!-- " + httpexpr + " -->\n" + result; - //if (ost.str() != fullresult) { - if (fullresult != ost.str()) { - cerr << "Parsing expression '" << expression << "', expected\n" - << fullresult << "\nbut got\n" << ost.str() << "\n"; - return false; - } - return true; -} - -bool test(const DocumentTypeRepo& repo, - const BucketIdFactory& bucketIdFactory) -{ - //std::cerr << "\n\nTESTING DOCUMENT SELECT PARSER\n\n"; - DocSelectionGrammar grammar(repo, bucketIdFactory); - - using boost::spirit::classic::space_p; - - // Parser two is the arithmvalue.. - // idspec, fieldspec, number & stringval, + - * / % () - testExpr(repo, bucketIdFactory, "3.14", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>3.14</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "-999", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>-999</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "15e4", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>15e4</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "3.4e-4", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>3.4e-4</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "\" Test \"", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"string\">\n" - " <parsenode rule=\"string\">\n" - " <value> Test </value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "id", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"idspec\">\n" - " <parsenode rule=\"idspec\">\n" - " <value>id</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "id.namespace", - grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"idspec\">\n" - " <parsenode rule=\"idspec\">\n" - " <value>id</value>\n" - " </parsenode>\n" - " <parsenode rule=\"idarg\">\n" - " <parsenode rule=\"idarg\">\n" - " <value>namespace</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "id.hash()", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"idspec\">\n" - " <parsenode rule=\"idspec\">\n" - " <value>id</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"function\">\n" - " <parsenode rule=\"function\">\n" - " <value>hash</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "id.namespace.hash()", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"idspec\">\n" - " <parsenode rule=\"idspec\">\n" - " <value>id</value>\n" - " </parsenode>\n" - " <parsenode rule=\"idarg\">\n" - " <parsenode rule=\"idarg\">\n" - " <value>namespace</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"function\">\n" - " <parsenode rule=\"function\">\n" - " <value>hash</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "music.artist", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"fieldspec\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>music</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"fieldname\">\n" - " <parsenode rule=\"fieldname\">\n" - " <value>artist</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "music.artist.lowercase()", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"fieldspec\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>music</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"fieldname\">\n" - " <parsenode rule=\"fieldname\">\n" - " <value>artist</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"function\">\n" - " <parsenode rule=\"function\">\n" - " <value>lowercase</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "(43)", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"valuegroup\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>43</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "1 + 2 * 3 - 10 % 2 / 3", grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"valuefuncadd\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>1</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncadd\">\n" - " <value>+</value>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>2</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <value>*</value>\n" - " </parsenode>\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>3</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncadd\">\n" - " <value>-</value>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <parsenode rule=\"valuefuncmod\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>10</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmod\">\n" - " <value>%</value>\n" - " </parsenode>\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>2</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <value>/</value>\n" - " </parsenode>\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>3</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "(43 + 14) / 34", - grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <parsenode rule=\"valuegroup\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"valuefuncadd\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>43</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncadd\">\n" - " <value>+</value>\n" - " </parsenode>\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>14</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <value>/</value>\n" - " </parsenode>\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>34</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "34 * (3 - 1) % 4", - grammar.use_parser<2>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>34</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmul\">\n" - " <value>*</value>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmod\">\n" - " <parsenode rule=\"valuegroup\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"valuefuncadd\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>3</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncadd\">\n" - " <value>-</value>\n" - " </parsenode>\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>1</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"valuefuncmod\">\n" - " <value>%</value>\n" - " </parsenode>\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>4</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - - // Parser 1 is a leaf. bool, comparison, fieldspec, doctype - testExpr(repo, bucketIdFactory, "true", grammar.use_parser<1>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>true</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "false", grammar.use_parser<1>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>false</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "music.test", grammar.use_parser<1>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"fieldspec\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>music</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"fieldname\">\n" - " <parsenode rule=\"fieldname\">\n" - " <value>test</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, "music", grammar.use_parser<1>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>music</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "music.artist = \"*john*\"", grammar.use_parser<1>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"comparison\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"fieldspec\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>music</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"fieldname\">\n" - " <parsenode rule=\"fieldname\">\n" - " <value>artist</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"operator\">\n" - " <parsenode rule=\"operator\">\n" - " <value>=</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"string\">\n" - " <parsenode rule=\"string\">\n" - " <value>*john*</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "music.length >= 180", grammar.use_parser<1>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"comparison\">\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"fieldspec\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>music</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"fieldname\">\n" - " <parsenode rule=\"fieldname\">\n" - " <value>length</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"operator\">\n" - " <parsenode rule=\"operator\">\n" - " <value>>=</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"arithmvalue\">\n" - " <parsenode rule=\"value\">\n" - " <parsenode rule=\"number\">\n" - " <parsenode rule=\"number\">\n" - " <value>180</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - - // Parser 0 - The whole expression - testExpr(repo, bucketIdFactory, - "true oR nOt false And true", grammar.use_parser<0>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"expression\">\n" - " <parsenode rule=\"or\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>true</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"or\">\n" - " <value>oR</value>\n" - " </parsenode>\n" - " <parsenode rule=\"and\">\n" - " <parsenode rule=\"not\">\n" - " <parsenode rule=\"not\">\n" - " <value>nOt</value>\n" - " </parsenode>\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>false</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"and\">\n" - " <value>And</value>\n" - " </parsenode>\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>true</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "(true oR false) aNd true", grammar.use_parser<0>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"expression\">\n" - " <parsenode rule=\"and\">\n" - " <parsenode rule=\"group\">\n" - " <parsenode rule=\"or\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>true</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"or\">\n" - " <value>oR</value>\n" - " </parsenode>\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>false</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"and\">\n" - " <value>aNd</value>\n" - " </parsenode>\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"bool\">\n" - " <parsenode rule=\"bool\">\n" - " <value>true</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - testExpr(repo, bucketIdFactory, - "iddoc or not(notand and ornot)", grammar.use_parser<0>(), - "<parsetree version=\"1.0\">\n" - " <parsenode rule=\"expression\">\n" - " <parsenode rule=\"or\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>iddoc</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"or\">\n" - " <value>or</value>\n" - " </parsenode>\n" - " <parsenode rule=\"not\">\n" - " <parsenode rule=\"not\">\n" - " <value>not</value>\n" - " </parsenode>\n" - " <parsenode rule=\"group\">\n" - " <parsenode rule=\"and\">\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>notand</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " <parsenode rule=\"and\">\n" - " <value>and</value>\n" - " </parsenode>\n" - " <parsenode rule=\"leaf\">\n" - " <parsenode rule=\"doctype\">\n" - " <parsenode rule=\"doctype\">\n" - " <value>ornot</value>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - " </parsenode>\n" - "</parsetree>\n"); - return true; -} - -} - -vespalib::Lock Parser::_G_parseLock; - -unique_ptr<Node> Parser::parse(const vespalib::stringref & s) -{ - - simple::SelectionParser simple(_bucketIdFactory); - if (simple.parse(s) && simple.getRemaining().empty()) { - Node::UP tmp(simple.getNode()); - assert(tmp.get() != NULL); - return tmp; - } else { - return fullParse(s); - } -} - -unique_ptr<Node> Parser::fullParse(const vespalib::stringref & s) -{ - static bool haveTested = test(_repo, _bucketIdFactory); if (haveTested) {} - try{ - vespalib::LockGuard guard(_G_parseLock); - DocSelectionGrammar grammar(_repo, _bucketIdFactory); - boost::spirit::classic::tree_parse_info<> info - = pt_parse(&s[0], &s[0]+s.size(), - grammar.use_parser<0>(), boost::spirit::classic::space_p); - if (!info.full) { - vespalib::string unexpected(info.stop); - unsigned int position = s.size() - unexpected.size(); - if (unexpected.size() > 10) { - unexpected = unexpected.substr(0,10); - } - vespalib::asciistream ost; - ost << "Unexpected token at position " << position << " ('" - << unexpected << "') in query '" << s << "',"; - throw ParsingFailedException(ost.str(), VESPA_STRLOC); - } - parse_assert(info.trees.size() == 1); - //printSpiritTree(std::cerr, info.trees, s, grammar); - return parseTree(grammar, info.trees[0]); - } catch (ParsingFailedException& e) { - throw; - } catch (vespalib::Exception& e) { - throw ParsingFailedException("Parsing failed. See cause exception.", - e, VESPA_STRLOC); - } catch (std::exception& e) { - cerr << "Parser::parse() internal error: " - << e.what() << endl; - throw; // Program will abort when this tries to go out.. - } - return unique_ptr<Node>(); } -} // select -} // document diff --git a/document/src/vespa/document/select/parser.h b/document/src/vespa/document/select/parser.h index 4df00d64bf3..35d710298c3 100644 --- a/document/src/vespa/document/select/parser.h +++ b/document/src/vespa/document/select/parser.h @@ -1,37 +1,40 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #pragma once #include "node.h" +#include "parsing_failed_exception.h" #include <vespa/document/bucket/bucketidfactory.h> -#include <vespa/vespalib/util/exception.h> -#include <vespa/vespalib/util/sync.h> - -namespace document { -class DocumentTypeRepo; - -namespace select { - -VESPA_DEFINE_EXCEPTION(ParsingFailedException, vespalib::Exception); - +#include <vespa/document/repo/documenttyperepo.h> +#include <memory> +#include <string> + +namespace document::select { + +/** + * Document selection parser built around Flex/Bison. O(n) on input size + * and non-locking. + * + * Thread safety: same as a std::vector + */ class Parser { + const DocumentTypeRepo&_doc_type_repo; + const BucketIdFactory& _bucket_id_factory; public: - Parser(const DocumentTypeRepo&, const BucketIdFactory& bucketIdFactory); + Parser(const DocumentTypeRepo& repo, const BucketIdFactory& bucket_id_factory) + : _doc_type_repo(repo), + _bucket_id_factory(bucket_id_factory) + {} /** * Returns a newly allocated AST root node representing the selection * if parsing is successful. Otherwise, ParsingFailedException will be * thrown. + * + * Thread safe, assuming referenced DocumentTypeRepo and BucketIdFactory + * instances are immutable. */ - std::unique_ptr<Node> parse(const vespalib::stringref& s); - -private: - std::unique_ptr<Node> fullParse(const vespalib::stringref& s); - static vespalib::Lock _G_parseLock; - const DocumentTypeRepo& _repo; - const BucketIdFactory& _bucketIdFactory; + std::unique_ptr<Node> parse(const std::string& str) const; }; -} // select -} // parser +} diff --git a/document/src/vespa/document/select/parsing_failed_exception.cpp b/document/src/vespa/document/select/parsing_failed_exception.cpp new file mode 100644 index 00000000000..ce02389ed46 --- /dev/null +++ b/document/src/vespa/document/select/parsing_failed_exception.cpp @@ -0,0 +1,9 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "parsing_failed_exception.h" +#include <vespa/document/base/exceptions.h> + +namespace document::select { + +VESPA_IMPLEMENT_EXCEPTION(ParsingFailedException, vespalib::Exception); + +}
\ No newline at end of file diff --git a/document/src/vespa/document/select/parsing_failed_exception.h b/document/src/vespa/document/select/parsing_failed_exception.h new file mode 100644 index 00000000000..54138a492e8 --- /dev/null +++ b/document/src/vespa/document/select/parsing_failed_exception.h @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/exception.h> + +namespace document::select { + +VESPA_DEFINE_EXCEPTION(ParsingFailedException, vespalib::Exception); + +} diff --git a/document/src/vespa/document/select/scanner.h b/document/src/vespa/document/select/scanner.h new file mode 100644 index 00000000000..5aa9ea1c8d3 --- /dev/null +++ b/document/src/vespa/document/select/scanner.h @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#if !defined(yyFlexLexerOnce) +# include <FlexLexer.h> +#endif + +#include "parser.hxx" +#include "location.hh" +#include <iosfwd> + +namespace document::select { + +class DocSelScanner final : yyFlexLexer { +public: + explicit DocSelScanner(std::istream* in) : yyFlexLexer(in) {} + ~DocSelScanner() override = default; + int yylex(DocSelParser::semantic_type* yylval, DocSelParser::location_type* yyloc); +}; + +} diff --git a/document/src/vespa/document/select/traversingvisitor.cpp b/document/src/vespa/document/select/traversingvisitor.cpp index b8f34540b29..26de6093ddf 100644 --- a/document/src/vespa/document/select/traversingvisitor.cpp +++ b/document/src/vespa/document/select/traversingvisitor.cpp @@ -73,12 +73,6 @@ TraversingVisitor::visitIdValueNode(const IdValueNode &) void -TraversingVisitor::visitSearchColumnValueNode(const SearchColumnValueNode &) -{ -} - - -void TraversingVisitor::visitFieldValueNode(const FieldValueNode &) { } diff --git a/document/src/vespa/document/select/traversingvisitor.h b/document/src/vespa/document/select/traversingvisitor.h index 43d10cfcaa2..f8b0377b102 100644 --- a/document/src/vespa/document/select/traversingvisitor.h +++ b/document/src/vespa/document/select/traversingvisitor.h @@ -21,7 +21,6 @@ public: void visitInvalidConstant(const InvalidConstant &) override; void visitDocumentType(const DocType &) override; void visitIdValueNode(const IdValueNode &) override; - void visitSearchColumnValueNode(const SearchColumnValueNode &) override; void visitFieldValueNode(const FieldValueNode &) override; void visitFloatValueNode(const FloatValueNode &) override; void visitVariableValueNode(const VariableValueNode &) override; diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 479896f9124..a0bbde0d21b 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -15,7 +15,6 @@ #include <iomanip> #include <sys/time.h> - #include <vespa/log/log.h> LOG_SETUP(".document.select.valuenode"); @@ -61,10 +60,7 @@ InvalidValueNode::print(std::ostream& out, bool verbose, if (hadParentheses()) out << ')'; } -NullValueNode::NullValueNode(const vespalib::stringref & name) - : _name(name) -{ } - +NullValueNode::NullValueNode() {} void NullValueNode::visit(Visitor &visitor) const @@ -79,7 +75,7 @@ NullValueNode::print(std::ostream& out, bool verbose, { (void) verbose; (void) indent; if (hadParentheses()) out << '('; - out << _name; + out << "null"; if (hadParentheses()) out << ')'; } @@ -678,88 +674,6 @@ IdValueNode::print(std::ostream& out, bool verbose, if (hadParentheses()) out << ')'; } -SearchColumnValueNode::SearchColumnValueNode( - const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, int numColumns) - : _bucketIdFactory(bucketIdFactory), - _id(name), - _numColumns(numColumns), - _distribution(std::make_unique<BucketDistribution>(_numColumns, 16)) -{ -} - -int64_t -SearchColumnValueNode::getValue(const BucketId& id) const -{ - return _distribution->getColumn(id); -} - - -std::unique_ptr<Value> -SearchColumnValueNode::getValue(const Context& context) const -{ - if (context._doc != NULL) { - return getValue(context._doc->getId()); - } else if (context._docId != NULL) { - return getValue(*context._docId); - } else { - return getValue(context._docUpdate->getId()); - } -} - - -std::unique_ptr<Value> -SearchColumnValueNode::getValue(const DocumentId& id) const -{ - return std::unique_ptr<Value>(new IntegerValue( - getValue(_bucketIdFactory.getBucketId(id)), false)); -} - - -std::unique_ptr<Value> -SearchColumnValueNode::traceValue(const Context& context, - std::ostream &out) const -{ - if (context._doc != NULL) { - return traceValue(context._doc->getId(), out); - } else if (context._docId != NULL) { - return traceValue(*context._docId, out); - } else { - return traceValue(context._docUpdate->getId(), out); - } -} - - -std::unique_ptr<Value> -SearchColumnValueNode::traceValue(const DocumentId& id, - std::ostream& out) const -{ - std::unique_ptr<Value> result(new IntegerValue( - getValue(_bucketIdFactory.getBucketId(id)), false)); - out << "Resolved search column of doc \"" << id << "\" to " << *result - << "\n"; - return result; -} - - -void -SearchColumnValueNode::visit(Visitor &visitor) const -{ - visitor.visitSearchColumnValueNode(*this); -} - - -void -SearchColumnValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _id; - out << '.' << _numColumns; - if (hadParentheses()) out << ')'; -} - namespace { union HashUnion { unsigned char _key[16]; @@ -1176,5 +1090,43 @@ ArithmeticValueNode::print(std::ostream& out, bool verbose, if (hadParentheses()) out << ')'; } +std::unique_ptr<FieldValueNode> FieldExprNode::convert_to_field_value() const { + const auto& doctype = resolve_doctype(); + // FIXME deprecate manual post-parsing of field expressions in favor of + // actually using the structural parser in the way nature intended. + vespalib::string mangled_expression; + build_mangled_expression(mangled_expression); + return std::make_unique<FieldValueNode>(doctype, mangled_expression); +} + +std::unique_ptr<FunctionValueNode> FieldExprNode::convert_to_function_call() const { + // Right hand expr string contains function call, lhs contains field spec on which + // the function is to be invoked. + if ((_left_expr == nullptr) || (_left_expr->_left_expr == nullptr)) { + throw vespalib::IllegalArgumentException( + vespalib::make_string("Cannot call function '%s' directly on document type", _right_expr.c_str())); + } + auto lhs = _left_expr->convert_to_field_value(); + const auto& function_name = _right_expr; + return std::make_unique<FunctionValueNode>(function_name, std::move(lhs)); +} + +void FieldExprNode::build_mangled_expression(vespalib::string& dest) const { + // Leftmost node is doctype, which should not be emitted as part of mangled expression. + if (_left_expr && _left_expr->_left_expr) { + _left_expr->build_mangled_expression(dest); + dest.push_back('.'); + } + dest.append(_right_expr); +} + +const vespalib::string& FieldExprNode::resolve_doctype() const { + const auto* leftmost = this; + while (leftmost->_left_expr) { + leftmost = leftmost->_left_expr.get(); + } + return leftmost->_right_expr; +} + } diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h index 0464159b85f..4257516d227 100644 --- a/document/src/vespa/document/select/valuenodes.h +++ b/document/src/vespa/document/select/valuenodes.h @@ -35,9 +35,8 @@ public: class NullValueNode : public ValueNode { - vespalib::string _name; public: - NullValueNode(const vespalib::stringref & name); + NullValueNode(); std::unique_ptr<Value> getValue(const Context&) const override { return std::unique_ptr<Value>(new NullValue()); @@ -48,7 +47,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new NullValueNode(_name)); + return wrapParens(new NullValueNode()); } }; @@ -56,7 +55,7 @@ class StringValueNode : public ValueNode { vespalib::string _value; public: - StringValueNode(const vespalib::stringref & val); + explicit StringValueNode(const vespalib::stringref & val); const vespalib::string& getValue() const { return _value; } @@ -115,6 +114,7 @@ class VariableValueNode : public ValueNode { vespalib::string _value; public: + // TODO stringref VariableValueNode(const vespalib::string & variableName) : _value(variableName) {} const vespalib::string& getVariableName() const { return _value; } @@ -183,6 +183,59 @@ private: void initFieldPath(const DocumentType&) const; }; +class FunctionValueNode; + +// Only used by the parser to build a partial field expression. Never part of +// an AST tree returned to the caller. +class FieldExprNode final : public ValueNode { + std::unique_ptr<FieldExprNode> _left_expr; + vespalib::string _right_expr; +public: + explicit FieldExprNode(const vespalib::string& doctype) : _left_expr(), _right_expr(doctype) {} + FieldExprNode(std::unique_ptr<FieldExprNode> left_expr, vespalib::stringref right_expr) + : _left_expr(std::move(left_expr)), _right_expr(right_expr) + {} + FieldExprNode(const FieldExprNode &) = delete; + FieldExprNode & operator = (const FieldExprNode &) = delete; + FieldExprNode(FieldExprNode &&) = default; + FieldExprNode & operator = (FieldExprNode &&) = default; + ~FieldExprNode() = default; + + std::unique_ptr<FieldValueNode> convert_to_field_value() const; + std::unique_ptr<FunctionValueNode> convert_to_function_call() const; +private: + void build_mangled_expression(vespalib::string& dest) const; + const vespalib::string& resolve_doctype() const; + + // These are not used, can just return dummy values. + std::unique_ptr<Value> getValue(const Context& context) const override { + (void) context; + return std::unique_ptr<Value>(); + } + std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override { + (void) context; + (void) out; + return std::unique_ptr<Value>(); + } + void print(std::ostream& out, bool verbose, const std::string& indent) const override { + (void) out; + (void) verbose; + (void) indent; + } + void visit(Visitor& visitor) const override { + (void) visitor; + } + + ValueNode::UP clone() const override { + if (_left_expr) { + return wrapParens(new FieldExprNode(std::unique_ptr<FieldExprNode>( + static_cast<FieldExprNode*>(_left_expr->clone().release())), _right_expr)); + } else { + return wrapParens(new FieldExprNode(_right_expr)); + } + } +}; + class IdValueNode : public ValueNode { public: @@ -222,35 +275,6 @@ private: int _divisionBits; }; -class SearchColumnValueNode : public ValueNode -{ -public: - SearchColumnValueNode(const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, - int numColumns); - - int getColumns() { return _numColumns; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - std::unique_ptr<Value> getValue(const DocumentId& id) const; - std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; - std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; - - int64_t getValue(const BucketId& bucketId) const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new SearchColumnValueNode(_bucketIdFactory, _id, _numColumns)); -} - -private: - const BucketIdFactory& _bucketIdFactory; - vespalib::string _id; - int _numColumns; - std::unique_ptr<BucketDistribution> _distribution; -}; - class FunctionValueNode : public ValueNode { public: diff --git a/document/src/vespa/document/select/visitor.h b/document/src/vespa/document/select/visitor.h index c89f0f24a6f..762d47c7c35 100644 --- a/document/src/vespa/document/select/visitor.h +++ b/document/src/vespa/document/select/visitor.h @@ -70,9 +70,6 @@ public: visitIdValueNode(const IdValueNode &) = 0; virtual void - visitSearchColumnValueNode(const SearchColumnValueNode &) = 0; - - virtual void visitFieldValueNode(const FieldValueNode &) = 0; virtual void diff --git a/filedistribution/OWNERS b/filedistribution/OWNERS index 31af040f698..2faf7df0593 100644 --- a/filedistribution/OWNERS +++ b/filedistribution/OWNERS @@ -1 +1,2 @@ -bratseth +baldersheim +hmusum diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index b0bdaff518f..d9699b700d0 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -54,6 +54,10 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </dependency> </dependencies> <build> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java new file mode 100644 index 00000000000..759a859253e --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -0,0 +1,111 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +import com.google.common.io.ByteStreams; +import com.yahoo.log.LogLevel; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * Utility class for compressing and decompressing files used in a file reference + * + * @author hmusum + */ +public class CompressedFileReference { + + private static final Logger log = Logger.getLogger(CompressedFileReference.class.getName()); + private static final int recurseDepth = 100; + + public static File compress(File baseDir, List<File> inputFiles, File outputFile) throws IOException { + ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outputFile))); + createArchiveFile(archiveOutputStream, baseDir, inputFiles); + return outputFile; + } + + public static byte[] compress(File directory) throws IOException { + return compress(directory, Files.find(Paths.get(directory.getAbsolutePath()), + recurseDepth, + (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) + .map(Path::toFile).collect(Collectors.toList())); + } + + public static byte[] compress(File baseDir, List<File> inputFiles) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(out)); + createArchiveFile(archiveOutputStream, baseDir, inputFiles); + return out.toByteArray(); + } + + static void decompress(File inputFile, File outputDir) throws IOException { + log.log(LogLevel.DEBUG, "Decompressing '" + inputFile + "' into '" + outputDir + "'"); + ArchiveInputStream ais = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile))); + decompress(ais, outputDir); + ais.close(); + } + + private static void decompress(ArchiveInputStream archiveInputStream, File outputFile) throws IOException { + int entries = 0; + ArchiveEntry entry; + while ((entry = archiveInputStream.getNextEntry()) != null) { + log.log(LogLevel.DEBUG, "Unpacking " + entry.getName()); + File outFile = new File(outputFile, entry.getName()); + if (entry.isDirectory()) { + if (!(outFile.exists() && outFile.isDirectory())) { + log.log(LogLevel.DEBUG, "Creating dir: " + outFile.getAbsolutePath()); + if (!outFile.mkdirs()) { + log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); + } + } + } else { + // Create parent dir if necessary + File parent = new File(outFile.getParent()); + if (!parent.exists() && !parent.mkdirs()) { + log.log(LogLevel.WARNING, "Could not create dir " + parent.getAbsolutePath()); + } + FileOutputStream fos = new FileOutputStream(outFile); + ByteStreams.copy(archiveInputStream, fos); + fos.close(); + } + entries++; + } + if (entries == 0) { + log.log(LogLevel.WARNING, "Not able to read any entries from " + outputFile.getName()); + } + } + + private static void createArchiveFile(ArchiveOutputStream archiveOutputStream, File baseDir, List<File> inputFiles) throws IOException { + inputFiles.forEach(file -> { + try { + writeFileToTar(archiveOutputStream, baseDir, file); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + archiveOutputStream.close(); + } + + private static void writeFileToTar(ArchiveOutputStream taos, File baseDir, File file) throws IOException { + log.log(LogLevel.DEBUG, "Adding file to tar: " + baseDir.toPath().relativize(file.toPath()).toString()); + taos.putArchiveEntry(taos.createArchiveEntry(file, baseDir.toPath().relativize(file.toPath()).toString())); + ByteStreams.copy(new FileInputStream(file), taos); + taos.closeArchiveEntry(); + } +} + diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index fde410bc8d7..9fe5eec54ff 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -16,7 +16,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -66,8 +65,8 @@ public class FileDownloader { fileReferences.forEach(this::queueForDownload); } - public void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { - fileReferenceDownloader.receiveFile(fileReference, filename, content, xxHash); + void receiveFile(FileReferenceData fileReferenceData) { + fileReferenceDownloader.receiveFile(fileReferenceData); } double downloadStatus(FileReference fileReference) { @@ -85,10 +84,6 @@ public class FileDownloader { private Optional<File> getFileFromFileSystem(FileReference fileReference, File directory) { File[] files = directory.listFiles(); if (directory.exists() && directory.isDirectory() && files != null && files.length > 0) { - if (files.length != 1) { - throw new RuntimeException("More than one file in '" + fileReference.value() + - "', expected only one, unable to proceed"); - } File file = files[0]; if (!file.exists()) { throw new RuntimeException("File with reference '" + fileReference.value() + diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 036c3157998..1ac3a1bd7df 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; @@ -33,7 +33,7 @@ public class FileReceiver { private final File downloadDirectory; private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); - public FileReceiver(Supervisor supervisor, FileReferenceDownloader downloader, File downloadDirectory) { + FileReceiver(Supervisor supervisor, FileReferenceDownloader downloader, File downloadDirectory) { this.supervisor = supervisor; this.downloader = downloader; this.downloadDirectory = downloadDirectory; @@ -68,14 +68,15 @@ public class FileReceiver { .paramDesc(4, "error-description", "Error description.") .returnDesc(0, "ret", "0 if success, 1 if crc mismatch, 2 otherwise")); // Temporary method until we have chunking - methods.add(new Method(RECEIVE_METHOD, "ssxlis", "i", handler, "receiveFile") + methods.add(new Method(RECEIVE_METHOD, "sssxlis", "i", handler, "receiveFile") .methodDesc("receive file reference content") .paramDesc(0, "file reference", "file reference to download") .paramDesc(1, "filename", "filename") - .paramDesc(2, "content", "array of bytes") - .paramDesc(3, "hash", "xx64hash of the file content") - .paramDesc(4, "errorcode", "Error code. 0 if none") - .paramDesc(5, "error-description", "Error description.") + .paramDesc(2, "type", "'file' or 'compressed'") + .paramDesc(3, "content", "array of bytes") + .paramDesc(4, "hash", "xx64hash of the file content") + .paramDesc(5, "errorcode", "Error code. 0 if none") + .paramDesc(6, "error-description", "Error description.") .returnDesc(0, "ret", "0 if success, 1 otherwise")); return methods; } @@ -84,15 +85,16 @@ public class FileReceiver { public final void receiveFile(Request req) { FileReference fileReference = new FileReference(req.parameters().get(0).asString()); String filename = req.parameters().get(1).asString(); - byte[] content = req.parameters().get(2).asData(); - long xxhash = req.parameters().get(3).asInt64(); - int errorCode = req.parameters().get(4).asInt32(); - String errorDescription = req.parameters().get(5).asString(); + String type = req.parameters().get(2).asString(); + byte[] content = req.parameters().get(3).asData(); + long xxhash = req.parameters().get(4).asInt64(); + int errorCode = req.parameters().get(5).asInt32(); + String errorDescription = req.parameters().get(6).asString(); if (errorCode == 0) { // TODO: Remove when system test works log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'"); - receiveFile(fileReference, filename, content, xxhash); + receiveFile(new FileReferenceData(fileReference, filename, FileReferenceData.Type.valueOf(type), content, xxhash)); req.returnValues().add(new Int32Value(0)); } else { log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription); @@ -101,19 +103,30 @@ public class FileReceiver { } } - void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { - long xxHashFromContent = hasher.hash(ByteBuffer.wrap(content), 0); - if (xxHashFromContent != xxHash) - throw new RuntimeException("xxhash from content (" + xxHashFromContent + ") is not equal to xxhash in request (" + xxHash + ")"); + void receiveFile(FileReferenceData fileReferenceData) { + long xxHashFromContent = hasher.hash(ByteBuffer.wrap(fileReferenceData.content()), 0); + if (xxHashFromContent != fileReferenceData.xxhash()) + throw new RuntimeException("xxhash from content (" + xxHashFromContent + ") is not equal to xxhash in request (" + fileReferenceData.xxhash()+ ")"); - File fileReferenceDir = new File(downloadDirectory, fileReference.value()); + File fileReferenceDir = new File(downloadDirectory, fileReferenceData.fileReference().value()); + // file might be a directory (and then type is compressed) + File file = new File(fileReferenceDir, fileReferenceData.filename()); try { - File tempFile = new File(Files.createTempDirectory("downloaded").toFile(), filename); - Files.write(tempFile.toPath(), content); - Files.createDirectories(fileReferenceDir.toPath()); - File file = new File(fileReferenceDir, filename); - moveFileToDestination(tempFile, file); - downloader.completedDownloading(fileReference, file); + File tempFile = new File(Files.createTempDirectory("downloaded").toFile(), fileReferenceData.filename()); + Files.write(tempFile.toPath(), fileReferenceData.content()); + + // Unpack if necessary + if (fileReferenceData.type() == FileReferenceData.Type.compressed) { + File decompressedDir = Files.createTempDirectory("decompressed").toFile(); + log.log(LogLevel.DEBUG, "Compressed file, unpacking " + tempFile + " to " + decompressedDir); + CompressedFileReference.decompress(tempFile, decompressedDir); + moveFileToDestination(decompressedDir, fileReferenceDir); + } else { + log.log(LogLevel.DEBUG, "Uncompressed file, moving to " + file.getAbsolutePath()); + Files.createDirectories(fileReferenceDir.toPath()); + moveFileToDestination(tempFile, file); + } + downloader.completedDownloading(fileReferenceData.fileReference(), file); } catch (IOException e) { log.log(LogLevel.ERROR, "Failed writing file: " + e.getMessage(), e); throw new RuntimeException("Failed writing file: ", e); @@ -123,7 +136,7 @@ public class FileReceiver { private void moveFileToDestination(File tempFile, File destination) { try { Files.move(tempFile.toPath(), destination.toPath()); - log.log(LogLevel.INFO, "Data written to " + destination.getAbsolutePath()); + log.log(LogLevel.INFO, "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); } catch (FileAlreadyExistsException e) { // Don't fail if it already exists (we might get the file from several config servers when retrying, servers are down etc. // so it might be written already) @@ -135,14 +148,16 @@ public class FileReceiver { } } - @SuppressWarnings({"UnusedDeclaration"}) + @SuppressWarnings({"UnusedDeclaration"}) public final void receiveFileMeta(Request req) { log.info("Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); } + @SuppressWarnings({"UnusedDeclaration"}) public final void receiveFilePart(Request req) { log.info("Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); } + @SuppressWarnings({"UnusedDeclaration"}) public final void receiveFileEof(Request req) { log.info("Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java new file mode 100644 index 00000000000..6272390f5cb --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java @@ -0,0 +1,61 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import net.jpountz.xxhash.XXHashFactory; + +import java.nio.ByteBuffer; + +/** + * Utility class for a file reference with data and metadata + * + * @author hmusum + */ +public class FileReferenceData { + + public enum Type {file, compressed} + + + + private final FileReference fileReference; + private final String filename; + private final Type type; + private final byte[] content; + private final long xxhash; + + public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content) { + this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0)); + } + + public FileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { + this.fileReference = fileReference; + this.filename = filename; + this.type = type; + this.content = content; + this.xxhash = xxhash; + } + + public static FileReferenceData empty(FileReference fileReference, String filename) { + return new FileReferenceData(fileReference, filename, FileReferenceData.Type.file, new byte[0], 0); + } + + public FileReference fileReference() { + return fileReference; + } + + public String filename() { + return filename; + } + + public Type type() { + return type; + } + + public byte[] content() { + return content; + } + + public long xxhash() { + return xxhash; + } +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 4c9c37dd6da..b51a4b68212 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -89,8 +89,8 @@ public class FileReferenceDownloader { downloadQueue.add(fileReferenceDownload); } - void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { - fileReceiver.receiveFile(fileReference, filename, content, xxHash); + void receiveFile(FileReferenceData fileReferenceData) { + fileReceiver.receiveFile(fileReferenceData); } synchronized Set<FileReference> queuedDownloads() { diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 278c46dab8b..1c9e8cdb91b 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -13,15 +13,13 @@ import com.yahoo.jrt.Transport; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; -import net.jpountz.xxhash.XXHash64; -import net.jpountz.xxhash.XXHashFactory; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -35,8 +33,6 @@ import static org.junit.Assert.fail; public class FileDownloaderTest { - private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); - private MockConnection connection; private FileDownloader fileDownloader; private File downloadDir; @@ -102,7 +98,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; - receiveFile(fileReference, filename, "some other content"); + receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); @@ -113,6 +109,40 @@ public class FileDownloaderTest { // Verify download status when downloaded assertDownloadStatus(fileDownloader, fileReference, 100.0); } + + { + // fileReference does not exist on disk, needs to be downloaded, is compressed data + + FileReference fileReference = new FileReference("fileReferenceToDirWithManyFiles"); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + + // Verify download status + assertDownloadStatus(fileDownloader, fileReference, 0.0); + + // Receives fileReference, should return and make it available to caller + String filename = "abc.tar.gz"; + Path tempPath = Files.createTempDirectory("dir"); + File subdir = new File(tempPath.toFile(), "subdir"); + File fooFile = new File(subdir, "foo"); + IOUtils.writeFile(fooFile, "foo", false); + File barFile = new File(subdir, "bar"); + IOUtils.writeFile(barFile, "bar", false); + + File tarFile = CompressedFileReference.compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename)); + byte[] tarredContent = IOUtils.readFileBytes(tarFile); + receiveFile(fileReference, filename, FileReferenceData.Type.compressed, tarredContent); + Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + + assertTrue(downloadedFile.isPresent()); + File downloadedFoo = new File(fileReferenceFullPath, tempPath.relativize(fooFile.toPath()).toString()); + File downloadedBar = new File(fileReferenceFullPath, tempPath.relativize(barFile.toPath()).toString()); + assertEquals("foo", IOUtils.readFile(downloadedFoo)); + assertEquals("bar", IOUtils.readFile(downloadedBar)); + + // Verify download status when downloaded + assertDownloadStatus(fileDownloader, fileReference, 100.0); + } } @Test @@ -133,7 +163,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; - receiveFile(fileReference, filename, "some other content"); + receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); @@ -168,7 +198,7 @@ public class FileDownloaderTest { public void receiveFile() throws IOException { FileReference foo = new FileReference("foo"); String filename = "foo.jar"; - receiveFile(foo, filename, "content"); + receiveFile(foo, filename, FileReferenceData.Type.file, "content"); File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename); assertEquals("content", IOUtils.readFile(downloadedFile)); } @@ -187,10 +217,12 @@ public class FileDownloaderTest { assertEquals(expectedDownloadStatus, downloadStatus, 0.0001); } - private void receiveFile(FileReference fileReference, String filename, String content) { - byte[] contentBytes = Utf8.toBytes(content); - long xxHashFromContent = hasher.hash(ByteBuffer.wrap(contentBytes), 0); - fileDownloader.receiveFile(fileReference, filename, contentBytes, xxHashFromContent); + private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, String content) { + receiveFile(fileReference, filename, type, Utf8.toBytes(content)); + } + + private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) { + fileDownloader.receiveFile(new FileReferenceData(fileReference, filename, type, content)); } private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { diff --git a/parent/pom.xml b/parent/pom.xml index 48d35410fff..0a19d66d674 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -22,20 +22,23 @@ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> </license> </licenses> - <developers> <developer> <name>Vespa</name> <url>https://github.com/vespa-engine</url> </developer> </developers> - <distributionManagement> <repository> <id>bintray-vespa-repo</id> <url>https://api.bintray.com/maven/yahoo/maven/vespa;publish=1</url> </repository> </distributionManagement> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> <repositories> <!-- Required for Athenz libraries --> @@ -49,12 +52,6 @@ </repository> </repositories> - <scm> - <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> - <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> - <url>git@github.com:vespa-engine/vespa.git</url> - </scm> - <build> <finalName>${project.artifactId}</finalName> <extensions> diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index cea72a556cc..4ec5bb928f6 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -618,8 +618,6 @@ TEST_F("Test that multiplication has higher priority than addition", TEST_F("Test that toplevel functions are visited", TestFixture) { - f.testPrune("searchcolumn.15 == 4", - "searchcolumn.15 == 4"); f.testPrune("id.scheme == \"doc\"", "id.scheme == \"doc\""); f.testPrune("test.aa < now() - 7200", diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 87efd74659d..d3567da3527 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -666,6 +666,42 @@ TEST("require that maintenance controller should change if some config has chang TEST_DO(assertMaintenanceControllerShouldChange(CCR().setMaintenanceChanged(true))); } +void +assertSubDbsShouldNotChange(DocumentDBConfig::ComparisonResult result) +{ + ReconfigParams params(result); + EXPECT_FALSE(params.configHasChanged()); + EXPECT_FALSE(params.shouldSubDbsChange()); +} + +void +assertSubDbsShouldChange(DocumentDBConfig::ComparisonResult result) +{ + ReconfigParams params(result); + EXPECT_TRUE(params.configHasChanged()); + EXPECT_TRUE(params.shouldSubDbsChange()); +} + + +TEST("require that subdbs should change if relevant config changed") +{ + TEST_DO(assertSubDbsShouldNotChange(CCR())); + EXPECT_FALSE(ReconfigParams(CCR().setMaintenanceChanged(true)).shouldSubDbsChange()); + TEST_DO(assertSubDbsShouldChange(CCR().setFlushChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setStoreChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setDocumenttypesChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setDocumentTypeRepoChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setSummaryChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setSummarymapChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setJuniperrcChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setAttributesChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setImportedFieldsChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setVisibilityDelayChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setRankProfilesChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setRankingConstantsChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setSchemaChanged(true))); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index b84aa1c1c6c..0f69684373c 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -497,7 +497,8 @@ public: _mcCfg->getLidSpaceCompactionConfig(), _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -514,7 +515,8 @@ public: _mcCfg->getLidSpaceCompactionConfig(), _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -531,7 +533,8 @@ public: _mcCfg->getLidSpaceCompactionConfig(), _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -546,7 +549,8 @@ public: cfg, _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index c6c810ae72f..f46f6c67fd4 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -143,9 +143,15 @@ index.warmup.time double default=0.0 restart # Indicate if we also want warm up with full unpack, instead of only cheaper seek. index.warmup.unpack bool default=false restart -## How many flushed indexes there can be befor fusion is forced. +## How many flushed indexes there can be befor fusion is forced while node is +## not in retired state. ## Setting to 1 will force an immediate fusion. -index.maxflushed int default=2 restart +index.maxflushed int default=2 + +## How many flushed indexes there can be befor fusion is forced while node is +## in retired state. +## Setting to 1 will force an immediate fusion. +index.maxflushedretired int default=20 ## How much memory is set aside for caching. ## Now only used for caching of dictionary lookups. diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp index 0915c11acfa..da45953e7b9 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp @@ -21,7 +21,6 @@ using document::select::Or; using document::select::ArithmeticValueNode; using document::select::FunctionValueNode; using document::select::IdValueNode; -using document::select::SearchColumnValueNode; using document::select::FieldValueNode; using document::select::FloatValueNode; using document::select::VariableValueNode; @@ -235,7 +234,7 @@ SelectPruner::visitDocumentType(const DocType &expr) _inverted = true; res = !res; } - _node.reset(new Constant(res ? "true" : "false")); + _node.reset(new Constant(res)); _resultSet.add(res ? Result::True : Result::False); _priority = DocumentTypePriority; } @@ -411,7 +410,7 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr) if (!_hasFields) { // If we're working on removed document sub db then we have no fields. _constVal = true; - _valueNode.reset(new NullValueNode("null")); + _valueNode.reset(new NullValueNode()); _priority = NullValPriority; return; } @@ -511,7 +510,7 @@ SelectPruner::setTernaryConst(bool val) { _constVal = true; _priority = ConstPriority; - _node.reset(new Constant(val ? "true" : "false")); + _node.reset(new Constant(val)); } void diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 00dd4befed7..1607c19611a 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -115,6 +115,9 @@ public: virtual void setSchema(const Schema &schema, SerialNum serialNum) override { _maintainer.setSchema(schema, serialNum); } + virtual void setMaxFlushed(uint32_t maxFlushed) override { + _maintainer.setMaxFlushed(maxFlushed); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 508a0208e70..8ee6a4f73ae 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(searchcore_server STATIC disk_mem_usage_forwarder.cpp docstorevalidator.cpp document_db_explorer.cpp + document_db_flush_config.cpp document_db_maintenance_config.cpp document_meta_store_read_guards.cpp document_scan_iterator.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.cpp new file mode 100644 index 00000000000..d11b8ee397d --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.cpp @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "document_db_maintenance_config.h" + +namespace proton { + +DocumentDBFlushConfig::DocumentDBFlushConfig() + : DocumentDBFlushConfig(2, 20) +{ +} + +DocumentDBFlushConfig::DocumentDBFlushConfig(uint32_t maxFlushed, uint32_t maxFlushedRetired) + : _maxFlushed(maxFlushed), + _maxFlushedRetired(maxFlushedRetired) +{ +} + +bool +DocumentDBFlushConfig::operator==(const DocumentDBFlushConfig &rhs) const +{ + return _maxFlushed == rhs._maxFlushed && + _maxFlushedRetired == rhs._maxFlushedRetired; +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.h new file mode 100644 index 00000000000..316d0716477 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.h @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace proton { + +/* + * Flush config, used to adjust flush targets as needed. + */ +class DocumentDBFlushConfig { + uint32_t _maxFlushed; + uint32_t _maxFlushedRetired; + +public: + DocumentDBFlushConfig(); + DocumentDBFlushConfig(uint32_t maxFlushed, uint32_t maxFlushedRetired); + bool operator==(const DocumentDBFlushConfig &rhs) const; + uint32_t getMaxFlushed() const { return _maxFlushed; } + uint32_t getMaxFlushedRetired() const { return _maxFlushedRetired; } +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp index cc491230784..848b1f27574 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp @@ -118,7 +118,8 @@ DocumentDBMaintenanceConfig::DocumentDBMaintenanceConfig() _lidSpaceCompaction(), _attributeUsageFilterConfig(), _attributeUsageSampleInterval(60.0), - _blockableJobConfig() + _blockableJobConfig(), + _flushConfig() { } @@ -131,7 +132,8 @@ DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig & const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction, const AttributeUsageFilterConfig &attributeUsageFilterConfig, double attributeUsageSampleInterval, - const BlockableMaintenanceJobConfig &blockableJobConfig) + const BlockableMaintenanceJobConfig &blockableJobConfig, + const DocumentDBFlushConfig &flushConfig) : _pruneRemovedDocuments(pruneRemovedDocuments), _heartBeat(heartBeat), _sessionCachePruneInterval(groupingSessionPruneInterval), @@ -139,7 +141,8 @@ DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig & _lidSpaceCompaction(lidSpaceCompaction), _attributeUsageFilterConfig(attributeUsageFilterConfig), _attributeUsageSampleInterval(attributeUsageSampleInterval), - _blockableJobConfig(blockableJobConfig) + _blockableJobConfig(blockableJobConfig), + _flushConfig(flushConfig) { } @@ -154,7 +157,8 @@ operator==(const DocumentDBMaintenanceConfig &rhs) const _lidSpaceCompaction == rhs._lidSpaceCompaction && _attributeUsageFilterConfig == rhs._attributeUsageFilterConfig && _attributeUsageSampleInterval == rhs._attributeUsageSampleInterval && - _blockableJobConfig == rhs._blockableJobConfig; + _blockableJobConfig == rhs._blockableJobConfig && + _flushConfig == rhs._flushConfig; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 2d4249531ec..acbbc442c7a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter_config.h> #include <vespa/fastos/timestamp.h> +#include "document_db_flush_config.h" namespace proton { @@ -95,6 +96,7 @@ private: AttributeUsageFilterConfig _attributeUsageFilterConfig; double _attributeUsageSampleInterval; BlockableMaintenanceJobConfig _blockableJobConfig; + DocumentDBFlushConfig _flushConfig; public: DocumentDBMaintenanceConfig(); @@ -106,7 +108,8 @@ public: const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction, const AttributeUsageFilterConfig &attributeUsageFilterConfig, double attributeUsageSampleInterval, - const BlockableMaintenanceJobConfig &blockableJobConfig); + const BlockableMaintenanceJobConfig &blockableJobConfig, + const DocumentDBFlushConfig &flushConfig); bool operator==(const DocumentDBMaintenanceConfig &rhs) const; @@ -133,6 +136,7 @@ public: const BlockableMaintenanceJobConfig &getBlockableJobConfig() const { return _blockableJobConfig; } + const DocumentDBFlushConfig &getFlushConfig() const { return _flushConfig; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp index 2eee5dff09d..5693e56ccf2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp @@ -12,7 +12,8 @@ DocumentSubDbInitializerResult::DocumentSubDbInitializerResult() _summaryManager(std::make_shared<SummaryManager::SP>()), _attributeManager(std::make_shared<AttributeManager::SP>()), _indexManager(std::make_shared<IIndexManager::SP>()), - _lidReuseDelayerConfig() + _lidReuseDelayerConfig(), + _flushConfig() { } @@ -23,5 +24,12 @@ setLidReuseDelayerConfig(LidReuseDelayerConfig lidReuseDelayerConfig_in) _lidReuseDelayerConfig = lidReuseDelayerConfig_in; } +void +DocumentSubDbInitializerResult:: +setFlushConfig(const DocumentDBFlushConfig &flushConfig) +{ + _flushConfig = flushConfig; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h index 67b524332eb..da628c1ab5d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h> #include <vespa/searchcorespi/index/iindexmanager.h> #include <vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h> +#include "document_db_flush_config.h" namespace proton { @@ -24,6 +25,7 @@ private: std::shared_ptr<searchcorespi::IIndexManager::SP> _indexManager; using LidReuseDelayerConfig = documentmetastore::LidReuseDelayerConfig; LidReuseDelayerConfig _lidReuseDelayerConfig; + DocumentDBFlushConfig _flushConfig; public: DocumentSubDbInitializerResult(); @@ -56,6 +58,8 @@ public: const LidReuseDelayerConfig &lidReuseDelayerConfig() const { return _lidReuseDelayerConfig; } + void setFlushConfig(const DocumentDBFlushConfig &flushConfig); + const DocumentDBFlushConfig &getFlushConfig() const { return _flushConfig; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index 44f4260f7f8..b83040b04ff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -42,7 +42,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() schemaChanged(false), maintenanceChanged(false), storeChanged(false), - visibilityDelayChanged(false) + visibilityDelayChanged(false), + flushChanged(false) { } DocumentDBConfig::DocumentDBConfig( @@ -152,6 +153,7 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const retval.maintenanceChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get()); retval.storeChanged = (_storeConfig != rhs._storeConfig); retval.visibilityDelayChanged = (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay()); + retval.flushChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get(), [](const auto &l, const auto &r) { return l.getFlushConfig() == r.getFlushConfig(); }); return retval; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 4250ce61175..cfca754a3d6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -46,6 +46,7 @@ public: bool maintenanceChanged; bool storeChanged; bool visibilityDelayChanged; + bool flushChanged; ComparisonResult(); ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; } @@ -63,7 +64,20 @@ public: ComparisonResult &setMaintenanceChanged(bool val) { maintenanceChanged = val; return *this; } ComparisonResult &setStoreChanged(bool val) { storeChanged = val; return *this; } - ComparisonResult &setVisibilityDelayChanged(bool val) { visibilityDelayChanged = val; return *this; } + ComparisonResult &setVisibilityDelayChanged(bool val) { + visibilityDelayChanged = val; + if (val) { + maintenanceChanged = true; + } + return *this; + } + ComparisonResult &setFlushChanged(bool val) { + flushChanged = val; + if (val) { + maintenanceChanged = true; + } + return *this; + } }; using SP = std::shared_ptr<DocumentDBConfig>; @@ -117,6 +131,14 @@ private: } return rhs != NULL && *lhs == *rhs; } + template <typename T, typename Func> + bool equals(const T *lhs, const T *rhs, Func isEqual) const + { + if (lhs == NULL) { + return rhs == NULL; + } + return rhs != NULL && isEqual(*lhs, *rhs); + } public: DocumentDBConfig(int64_t generation, const RankProfilesConfigSP &rankProfiles, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 6aea5234fbe..b9f59c4c411 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -136,7 +136,10 @@ buildMaintenanceConfig(const BootstrapConfig::SP &bootstrapConfig, proton.writefilter.sampleinterval, BlockableMaintenanceJobConfig( proton.maintenancejobs.resourcelimitfactor, - proton.maintenancejobs.maxoutstandingmoveops)); + proton.maintenancejobs.maxoutstandingmoveops), + DocumentDBFlushConfig( + proton.index.maxflushed, + proton.index.maxflushedretired)); } template<typename T> diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index c766cc89bb3..4c1dd47569b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -309,6 +309,15 @@ DocumentSubDBCollection::close() } void +DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc) +{ + _calc = calc; + for (auto subDb : _subDBs) { + subDb->setBucketStateCalculator(calc); + } +} + +void DocumentSubDBCollection::tearDownReferences(IDocumentDBReferenceResolver &resolver) { for (auto subDb : _subDBs) { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 7290250c59e..ea07e391f69 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -94,9 +94,7 @@ public: const HwInfo &hwInfo); ~DocumentSubDBCollection(); - void setBucketStateCalculator(const IBucketStateCalculatorSP &calc) { - _calc = calc; - } + void setBucketStateCalculator(const IBucketStateCalculatorSP &calc); void createRetrievers(); void maintenanceSync(MaintenanceController &mc, ICommitable &commit); diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 2251a24c58a..d41c2088518 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -25,6 +25,7 @@ class DocumentSubDbInitializerResult; class FeedHandler; class FileConfigManager; class IAttributeManager; +class IBucketStateCalculator; class IDcoumentRetriever; class IDocumentDBReferenceResolver; class IDocumentDBReference; @@ -72,6 +73,7 @@ public: virtual IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) = 0; + virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) = 0; virtual std::shared_ptr<ISearchHandler> getSearchView() const = 0; virtual std::shared_ptr<IFeedView> getFeedView() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index 377b012de99..8ec41ae3e3c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -64,7 +64,7 @@ bool ReconfigParams::shouldSubDbsChange() const { return shouldMatchersChange() || shouldAttributeManagerChange() || shouldSummaryManagerChange() - || _res.documentTypeRepoChanged || _res.documenttypesChanged || _res.storeChanged; + || _res.documentTypeRepoChanged || _res.documenttypesChanged || _res.storeChanged || _res.flushChanged; } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 4069a871210..e05ab4b798f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -5,6 +5,7 @@ #include "document_subdb_initializer.h" #include "reconfig_params.h" #include "i_document_subdb_owner.h" +#include "ibucketstatecalculator.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> #include <vespa/searchcore/proton/index/index_manager_initializer.h> @@ -49,7 +50,9 @@ SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx) getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()), _numSearcherThreads(cfg._numSearcherThreads), _warmupExecutor(ctx._warmupExecutor), - _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()) + _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()), + _flushConfig(), + _nodeRetired(false) { _gidToLidChangeHandler = _realGidToLidChangeHandler; } @@ -136,6 +139,7 @@ SearchableDocSubDB::setup(const DocumentSubDbInitializerResult &initResult) Parent::setup(initResult); setupIndexManager(initResult.indexManager()); _docIdLimit.set(_dms->getCommittedDocIdLimit()); + applyFlushConfig(initResult.getFlushConfig()); } void @@ -160,6 +164,7 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig()); IReprocessingTask::List tasks; updateLidReuseDelayer(&newConfigSnapshot); + applyFlushConfig(newConfigSnapshot.getMaintenanceConfigSP()->getFlushConfig()); if (params.shouldMatchersChange() && _addMetrics) { reconfigureMatchingMetrics(newConfigSnapshot.getRankProfilesConfig()); } @@ -185,6 +190,27 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const } void +SearchableDocSubDB::applyFlushConfig(const DocumentDBFlushConfig &flushConfig) +{ + _flushConfig = flushConfig; + propagateFlushConfig(); +} + +void +SearchableDocSubDB::propagateFlushConfig() +{ + uint32_t maxFlushed = _nodeRetired ? _flushConfig.getMaxFlushedRetired() : _flushConfig.getMaxFlushed(); + _indexMgr->setMaxFlushed(maxFlushed); +} + +void +SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) +{ + _nodeRetired = calc->nodeRetired(); + propagateFlushConfig(); +} + +void SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const SessionManager::SP &sessionManager) { assert(_writeService.master().isCurrentThread()); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 6da1a337cda..0852e028c25 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -85,6 +85,8 @@ private: const size_t _numSearcherThreads; vespalib::ThreadExecutor &_warmupExecutor; std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler; + DocumentDBFlushConfig _flushConfig; + bool _nodeRetired; // Note: lifetime of indexManager must be handled by caller. std::shared_ptr<initializer::InitializerTask> @@ -100,6 +102,8 @@ private: bool reconfigure(vespalib::Closure0<bool>::UP closure) override; void reconfigureIndexSearchable(); void syncViews(); + void applyFlushConfig(const DocumentDBFlushConfig &flushConfig); + void propagateFlushConfig(); protected: IFlushTargetList getFlushTargetsInternal() override; @@ -120,6 +124,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; + virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override; void clearViews() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index e78f8136d26..c5a8706a0ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -329,6 +329,7 @@ StoreOnlyDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot, Ser LidReuseDelayerConfig lidReuseDelayerConfig(configSnapshot); result->writableResult().setLidReuseDelayerConfig(lidReuseDelayerConfig); + result->writableResult().setFlushConfig(configSnapshot.getMaintenanceConfigSP()->getFlushConfig()); return result; } @@ -454,6 +455,11 @@ StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config) _rSummaryMgr->reconfigure(config); } +void +StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &) +{ +} + proton::IAttributeManager::SP StoreOnlyDocSubDB::getAttributeManager() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index f5038c252a7..d7b242f27ff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -215,6 +215,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; + virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override; ISearchHandler::SP getSearchView() const override { return _iSearchView.get(); } IFeedView::SP getFeedView() const override { return _iFeedView.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 6620795483d..66ae296566f 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -55,6 +55,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB { return IReprocessingTask::List(); } + void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &) override { } ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); } IFeedView::SP getFeedView() const override { return IFeedView::SP(); } void clearViews() override {} diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 9e26b1250ff..9e105ef8a7f 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -28,6 +28,7 @@ struct MockIndexManager : public searchcorespi::IIndexManager } virtual void setSchema(const Schema &, SerialNum) override {} virtual void heartBeat(SerialNum) override {} + virtual void setMaxFlushed(uint32_t) override { } }; } // namespace test diff --git a/searchcorespi/src/tests/plugin/plugin.cpp b/searchcorespi/src/tests/plugin/plugin.cpp index bea732f3db6..5614ea1da3a 100644 --- a/searchcorespi/src/tests/plugin/plugin.cpp +++ b/searchcorespi/src/tests/plugin/plugin.cpp @@ -35,6 +35,7 @@ public: return l; } virtual void setSchema(const Schema &, SerialNum) override { } + virtual void setMaxFlushed(uint32_t) override { } }; class IndexManagerFactory : public searchcorespi::IIndexManagerFactory diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h index b6538bd1ade..c529fe01dee 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h @@ -157,6 +157,13 @@ public: * @param schema The new schema to start using. **/ virtual void setSchema(const Schema &schema, SerialNum serialNum) = 0; + + /* + * Sets the max number of flushed indexes before fusion is urgent. + * + * @param maxFlushed The max number of flushed indexes before fusion is urgent. + */ + virtual void setMaxFlushed(uint32_t maxFlushed) = 0; }; } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index d18ff417074..a18d24931cb 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -1085,9 +1085,9 @@ IndexMaintainer::getFusionStats() const { LockGuard lock(_new_search_lock); source_list = _source_list; + stats.maxFlushed = _maxFlushed; } stats.diskUsage = source_list->getSearchableStats().sizeOnDisk(); - stats.maxFlushed = _maxFlushed; { LockGuard guard(_fusion_lock); stats.numUnfused = _fusion_spec.flush_ids.size() + ((_fusion_spec.last_fusion_id != 0) ? 1 : 0); @@ -1228,5 +1228,12 @@ IndexMaintainer::pruneRemovedFields(const Schema &schema, SerialNum serialNum) } } +void +IndexMaintainer::setMaxFlushed(uint32_t maxFlushed) +{ + LockGuard lock(_new_search_lock); + _maxFlushed = maxFlushed; +} + } // namespace index } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 133784f3b57..fb0bc2e5e78 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -383,6 +383,7 @@ public: IFlushTarget::List getFlushTargets() override; void setSchema(const Schema & schema, SerialNum serialNum) override ; + void setMaxFlushed(uint32_t maxFlushed) override; }; } // namespace index diff --git a/staging_vespalib/src/tests/metrics/mock_tick.cpp b/staging_vespalib/src/tests/metrics/mock_tick.cpp index e78b7c4644f..68e8d7a7daa 100644 --- a/staging_vespalib/src/tests/metrics/mock_tick.cpp +++ b/staging_vespalib/src/tests/metrics/mock_tick.cpp @@ -3,69 +3,4 @@ namespace vespalib::metrics { -const std::chrono::seconds oneSec{1}; - -TimeStamp -MockTick::next(TimeStamp prev) -{ - std::unique_lock<std::mutex> locker(_lock); - _prevValue = prev; - while (_runFlag) { - if (_provided) { - _blocked.store(false); - _provided.store(false); - return _nextValue; - } - _blocked.store(true); - _blockedCond.notify_all(); - auto r = _providedCond.wait_for(locker, oneSec); - (void)r; - } - return TimeStamp(0); -} - -void -MockTick::kill() -{ - std::unique_lock<std::mutex> locker(_lock); - _runFlag.store(false); - _blockedCond.notify_all(); - _providedCond.notify_all(); -} - -void -MockTick::provide(TimeStamp value) -{ - std::unique_lock<std::mutex> locker(_lock); - _nextValue = value; - _blocked.store(false); - _provided.store(true); - _providedCond.notify_all(); -} - -TimeStamp -MockTick::waitUntilBlocked() -{ - std::unique_lock<std::mutex> locker(_lock); - while (_runFlag) { - if (_blocked) { - return _prevValue; - } - auto r = _blockedCond.wait_for(locker, oneSec); - (void)r; - } - return TimeStamp(0); -} - -MockTick::MockTick() - : _lock(), - _runFlag(true), - _provided(false), - _blocked(false), - _providedCond(), - _blockedCond(), - _nextValue(0.0), - _prevValue(0.0) -{} - } // namespace vespalib::metrics diff --git a/staging_vespalib/src/tests/metrics/mock_tick.h b/staging_vespalib/src/tests/metrics/mock_tick.h index 2f69f69fc91..3f244ea6c9f 100644 --- a/staging_vespalib/src/tests/metrics/mock_tick.h +++ b/staging_vespalib/src/tests/metrics/mock_tick.h @@ -5,34 +5,87 @@ #include <condition_variable> #include <mutex> #include <vespa/vespalib/metrics/clock.h> +#include <vespa/vespalib/testkit/test_kit.h> namespace vespalib::metrics { -// share the MockTick between the tested and the tester. +// used to test clients of the Tick interface +// values shared between threads are bounded queues with max size 1 class MockTick : public Tick { private: - std::mutex _lock; - std::atomic<bool> _runFlag; - std::atomic<bool> _provided; - std::atomic<bool> _blocked; - std::condition_variable _providedCond; - std::condition_variable _blockedCond; - TimeStamp _nextValue; - TimeStamp _prevValue; -public: - MockTick(); - TimeStamp next(TimeStamp prev) override; - void kill() override; + using Guard = std::unique_lock<std::mutex>; + struct Value { + TimeStamp value{0.0}; + bool valid{false}; + }; + + TimeStamp _first_value; + std::mutex _lock; + std::condition_variable _cond; + bool _alive; + Value _prev; + Value _next; + + void push(Value &dst, TimeStamp value) { + Guard guard(_lock); + while (_alive && dst.valid) { + _cond.wait(guard); + } + dst.value = value; + dst.valid = true; + _cond.notify_one(); + } + + TimeStamp pop(Value &src) { + Guard guard(_lock); + while (_alive && !src.valid) { + _cond.wait(guard); + } + src.valid = false; + _cond.notify_one(); + return src.value; + } - void provide(TimeStamp value); - TimeStamp waitUntilBlocked(); + TimeStamp peek(const Value &src) { + Guard guard(_lock); + while (_alive && !src.valid) { + _cond.wait(guard); + } + return src.value; + } + +public: + MockTick(TimeStamp first_value) + : _first_value(first_value), _lock(), _cond(), _alive(true), _prev(), _next() {} + TimeStamp first() override { return _first_value; } + TimeStamp next(TimeStamp prev) override { + push(_prev, prev); + return pop(_next); + } + TimeStamp give(TimeStamp next_value) { + TimeStamp prev_value = pop(_prev); + push(_next, next_value); + EXPECT_EQUAL(peek(_prev).count(), next_value.count()); + return prev_value; + } + bool alive() const override { return _alive; } + void kill() override { + Guard guard(_lock); + _alive = false; + _cond.notify_all(); + } }; -struct TickProxy : Tick { - std::shared_ptr<Tick> tick; - TickProxy(std::shared_ptr<Tick> tick_in) : tick(std::move(tick_in)) {} - TimeStamp next(TimeStamp prev) override { return tick->next(prev); } - void kill() override { tick->kill(); } +// share the MockTick between the tested and the tester. +class TickProxy : public Tick { +private: + std::shared_ptr<Tick> _tick; +public: + TickProxy(std::shared_ptr<Tick> tick) : _tick(std::move(tick)) {} + TimeStamp first() override { return _tick->first(); } + TimeStamp next(TimeStamp prev) override { return _tick->next(prev); } + bool alive() const override { return _tick->alive(); } + void kill() override { _tick->kill(); } }; } // namespace vespalib::metrics diff --git a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp b/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp index 80030cb89b2..ce518474a7f 100644 --- a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp +++ b/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp @@ -90,24 +90,24 @@ fprintf(stderr, "bad json b\n"); void check_json(const vespalib::string &actual) { vespalib::string expect = "{" - " \"snapshot\": { \"from\": 1, \"to\": 4 }," - " \"values\": [ { \"name\": \"foo\"," - " \"values\": { \"count\": 17, \"rate\": 4.85714 }" + " snapshot: { from: 1, to: 4 }," + " values: [ { name: 'foo'," + " values: { count: 17, rate: 4.85714 }" " }, {" - " \"name\": \"foo\"," - " \"dimensions\": { \"chain\": \"default\", \"documenttype\": \"music\", \"thread\": \"0\" }," - " \"values\": { \"count\": 4, \"rate\": 1.14286 }" + " name: 'foo'," + " dimensions: { chain: 'default', documenttype: 'music', thread: '0' }," + " values: { count: 4, rate: 1.14286 }" " }, {" - " \"name\": \"bar\"," - " \"values\": { \"count\": 4, \"rate\": 1.14286, \"average\": 42, \"min\": 41, \"max\": 43, \"last\": 42 }" + " name: 'bar'," + " values: { count: 4, rate: 1.14286, average: 42, min: 41, max: 43, last: 42 }" " }, {" - " \"name\": \"bar\"," - " \"dimensions\": { \"chain\": \"vespa\", \"documenttype\": \"blogpost\", \"thread\": \"1\" }," - " \"values\": { \"count\": 1, \"rate\": 0.285714, \"average\": 14, \"min\": 14, \"max\": 14, \"last\": 14 }" + " name: 'bar'," + " dimensions: { chain: 'vespa', documenttype: 'blogpost', thread: '1' }," + " values: { count: 1, rate: 0.285714, average: 14, min: 14, max: 14, last: 14 }" " }, {" - " \"name\": \"bar\"," - " \"dimensions\": { \"chain\": \"vespa\", \"documenttype\": \"blogpost\", \"thread\": \"2\" }," - " \"values\": { \"count\": 1, \"rate\": 0.285714, \"average\": 11, \"min\": 11, \"max\": 11, \"last\": 11 }" + " name: 'bar'," + " dimensions: { chain: 'vespa', documenttype: 'blogpost', thread: '2' }," + " values: { count: 1, rate: 0.285714, average: 11, min: 11, max: 11, last: 11 }" " } ]" "}"; EXPECT_TRUE(compare_json(expect, actual)); @@ -119,10 +119,8 @@ TEST("use simple_metrics_collector") using namespace vespalib::metrics; SimpleManagerConfig cf; cf.sliding_window_seconds = 5; - std::shared_ptr<MockTick> ticker = std::make_shared<MockTick>(); - ticker->provide(TimeStamp(1.0)); + std::shared_ptr<MockTick> ticker = std::make_shared<MockTick>(TimeStamp(1.0)); auto manager = SimpleMetricsManager::createForTest(cf, std::make_unique<TickProxy>(ticker)); - EXPECT_EQUAL(1.0, ticker->waitUntilBlocked().count()); Counter myCounter = manager->counter("foo"); myCounter.add(); @@ -134,9 +132,7 @@ TEST("use simple_metrics_collector") myGauge.sample(43.0); myGauge.sample(42.0); - EXPECT_EQUAL(1.0, ticker->waitUntilBlocked().count()); - ticker->provide(TimeStamp(2.0)); - EXPECT_EQUAL(2.0, ticker->waitUntilBlocked().count()); + EXPECT_EQUAL(1.0, ticker->give(TimeStamp(2.0)).count()); Snapshot snap1 = manager->snapshot(); EXPECT_EQUAL(1.0, snap1.startTime()); @@ -178,9 +174,7 @@ TEST("use simple_metrics_collector") myGauge.sample(14.0, two); myGauge.sample(11.0, three); - EXPECT_EQUAL(2.0, ticker->waitUntilBlocked().count()); - ticker->provide(TimeStamp(4.5)); - EXPECT_EQUAL(4.5, ticker->waitUntilBlocked().count()); + EXPECT_EQUAL(2.0, ticker->give(TimeStamp(4.5)).count()); Snapshot snap2 = manager->snapshot(); EXPECT_EQUAL(1.0, snap2.startTime()); @@ -193,8 +187,7 @@ TEST("use simple_metrics_collector") // flush sliding window for (int i = 5; i <= 10; ++i) { - ticker->provide(TimeStamp(i)); - ticker->waitUntilBlocked(); + ticker->give(TimeStamp(i)); } Snapshot snap3 = manager->snapshot(); EXPECT_EQUAL(5.0, snap3.startTime()); diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.h b/staging_vespalib/src/vespa/vespalib/metrics/clock.h index d46c691b648..2af2ad89427 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/clock.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/clock.h @@ -21,8 +21,9 @@ using TimeStamp = std::chrono::duration<double, std::ratio<1,1>>; struct Tick { using UP = std::unique_ptr<Tick>; virtual TimeStamp next(TimeStamp prev) = 0; - TimeStamp first() { return next(TimeStamp(0.0)); } + virtual TimeStamp first() = 0; virtual void kill() = 0; + virtual bool alive() const = 0; virtual ~Tick() {} }; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter.h b/staging_vespalib/src/vespa/vespalib/metrics/counter.h index 1af61638d6b..36a25adda2d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/counter.h @@ -12,6 +12,9 @@ class MetricsManager; class CounterAggregator; +/** + * Represents a counter metric that can be incremented. + **/ class Counter { std::shared_ptr<MetricsManager> _manager; MetricName _id; @@ -30,8 +33,14 @@ public: void add(Point p) { add(1, p); } void add(size_t count) const { add(count, Point::empty); } + /** + * Increment the counter. + * @param count the amount to increment by (default 1) + * @param p the point representing labels for this increment (default empty) + **/ void add(size_t count, Point p) const; + // internal struct Increment { MetricIdentifier idx; size_t value; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge.h b/staging_vespalib/src/vespa/vespalib/metrics/gauge.h index 5f15a573b73..fa3f826f2b8 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/gauge.h @@ -11,6 +11,9 @@ namespace metrics { class MetricsManager; class GaugeAggregator; +/** + * Represents a gauge metric that can be measured. + **/ class Gauge { private: std::shared_ptr<MetricsManager> _manager; @@ -20,8 +23,14 @@ public: : _manager(std::move(m)), _id(id) {} + /** + * Provide a sample for the gauge. + * @param value the measurement for this sample + * @param p the point representing labels for this sample (default empty) + **/ void sample(double value, Point p = Point::empty) const; + // internal struct Measurement { MetricIdentifier idx; double value; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h index c1dc5302c1a..672b71766ae 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h @@ -9,6 +9,10 @@ namespace vespalib { namespace metrics { +/** + * utility for converting a snapshot to JSON format + * (which can be inserted into /state/v1/metrics page). + **/ class JsonFormatter { private: diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h index 020b44dc4b7..0fc5af41a81 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_identifier.h @@ -8,6 +8,7 @@ namespace vespalib { namespace metrics { +// internal struct MetricIdentifier { const MetricName _name; const Point _point; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h index 5bda0230bd4..5c764dcc74e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h @@ -8,6 +8,7 @@ namespace vespalib { namespace metrics { +// internal class for typechecking class MetricTypes { static const char *_typeNames[]; public: diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h index 2742252ed44..c86e5a315dc 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h @@ -14,31 +14,72 @@ #include "dimension.h" #include "label.h" -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { +/** + * Interface for a Metrics manager, for creating metrics + * and for fetching snapshots. + **/ class MetricsManager : public std::enable_shared_from_this<MetricsManager> { public: virtual ~MetricsManager() {} - virtual Counter counter(const vespalib::string &name) = 0; // get or create - virtual Gauge gauge (const vespalib::string &name) = 0; // get or create + /** + * Get or create a counter metric. + * @param name the name of the metric. + **/ + virtual Counter counter(const vespalib::string &name) = 0; + /** + * Get or create a gauge metric. + * @param name the name of the metric. + **/ + virtual Gauge gauge(const vespalib::string &name) = 0; + + /** + * Get or create a dimension for labeling metrics. + * @param name the name of the dimension. + **/ virtual Dimension dimension(const vespalib::string &name) = 0; // get or create + + /** + * Get or create a label. + * @param value the label value. + **/ virtual Label label(const vespalib::string &value) = 0; // get or create + + /** + * Create a PointBuilder for labeling metrics. + **/ PointBuilder pointBuilder() { return PointBuilder(shared_from_this()); } - virtual PointBuilder pointBuilder(Point from) = 0; - virtual Point pointFrom(PointMap::BackingMap map) = 0; + /** + * Create a PointBuilder for labeling metrics, starting with + * an Point of already existing dimension/label pairs, which can + * then be added to or changed. + * @param from provide a Point to start from. + * + **/ + virtual PointBuilder pointBuilder(Point from) = 0; + /** + * Create a snapshot of sampled metrics (usually for the last minute). + **/ virtual Snapshot snapshot() = 0; + + /** + * Create a snapshot of all sampled metrics the manager has seen. + **/ virtual Snapshot totalSnapshot() = 0; + // for use from PointBuilder only + virtual Point pointFrom(PointMap::BackingMap map) = 0; + // for use from Counter only virtual void add(Counter::Increment inc) = 0; @@ -48,4 +89,3 @@ public: } // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h index 6fd25552d6e..566fd2a3997 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h @@ -9,6 +9,7 @@ namespace vespalib { namespace metrics { +// internal class NameCollection { private: using Map = std::map<vespalib::string, size_t>; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h index 84d4c7fb569..83b804228dd 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h @@ -12,21 +12,42 @@ namespace metrics { class MetricsManager; +/** + * Build a Point for labeling metrics + **/ class PointBuilder { private: std::shared_ptr<MetricsManager> _owner; PointMap::BackingMap _map; public: + // for use from MetricsManager PointBuilder(std::shared_ptr<MetricsManager> m); PointBuilder(std::shared_ptr<MetricsManager> m, const PointMap::BackingMap &from); ~PointBuilder() {} + /** + * Bind a dimension to a label. + * Overwrites any label already bound to that dimension. + **/ PointBuilder &&bind(Dimension dimension, Label label) &&; + + /** + * Bind a dimension to a label. + * Convenience method that converts the label value. + **/ PointBuilder &&bind(Dimension dimension, LabelValue label) &&; + + /** + * Bind a dimension to a label. + * Convenience method that converts both the dimension name and the label value. + **/ PointBuilder &&bind(DimensionName dimension, LabelValue label) &&; + /** make a Point from the builder */ Point build(); + + /** make a Point from the builder */ operator Point () &&; }; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.h b/staging_vespalib/src/vespa/vespalib/metrics/point_map.h index 2810aa1aa9d..2ed50a85842 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_map.h @@ -8,6 +8,7 @@ namespace vespalib { namespace metrics { +// internal class PointMap { public: using BackingMap = std::map<Dimension, Label>; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h b/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h index 83891e7fa0c..ba301ff3f06 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h @@ -9,6 +9,7 @@ namespace vespalib { namespace metrics { +// internal class PointMapCollection { private: using PointMapMap = std::map<PointMap, size_t>; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp b/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp index a848628a4c0..ae8d6d76eac 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp @@ -22,8 +22,9 @@ Producer::getMetrics(const vespalib::string &) vespalib::string Producer::getTotalMetrics(const vespalib::string &) { - // not implemented - return ""; + Snapshot snap = _manager->totalSnapshot(); + JsonFormatter fmt(snap); + return fmt.asString(); } diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.h b/staging_vespalib/src/vespa/vespalib/metrics/producer.h index c1a9cae86ed..387d94379b7 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/producer.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/producer.h @@ -9,6 +9,9 @@ namespace metrics { class MetricsManager; +/** + * Utility class for wiring a MetricsManager into a StateApi. + **/ class Producer : public vespalib::MetricsProducer { private: std::shared_ptr<MetricsManager> _manager; diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h index 826a43f6713..6e6cff55b1f 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h @@ -5,3 +5,19 @@ #include <chrono> #include <memory> #include <vespa/vespalib/stllike/string.h> + +#include "clock.h" +#include "counter.h" +#include "dimension.h" +#include "dummy_metrics_manager.h" +#include "gauge.h" +#include "json_formatter.h" +#include "label.h" +#include "metric_identifier.h" +#include "metric_name.h" +#include "metrics_manager.h" +#include "point_builder.h" +#include "point.h" +#include "producer.h" +#include "simple_metrics_manager.h" +#include "snapshots.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp index 90c9ef0eab6..b9b3415a0e3 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp @@ -25,7 +25,6 @@ SimpleMetricsManager::SimpleMetricsManager(const SimpleManagerConfig &config, _firstBucket(0), _maxBuckets(config.sliding_window_seconds), _totalsBucket(0, _startTime, _startTime), - _runFlag(true), _thread(&SimpleMetricsManager::tickerLoop, this) { if (_maxBuckets < 1) _maxBuckets = 1; @@ -35,7 +34,6 @@ SimpleMetricsManager::SimpleMetricsManager(const SimpleManagerConfig &config, SimpleMetricsManager::~SimpleMetricsManager() { - _tickSupplier->kill(); stopThread(); } @@ -199,16 +197,18 @@ SimpleMetricsManager::pointFrom(PointMap::BackingMap map) void SimpleMetricsManager::tickerLoop() { - while (_runFlag) { + while (_tickSupplier->alive()) { TimeStamp now = _tickSupplier->next(_curTime); - tick(now); + if (_tickSupplier->alive()) { + tick(now); + } } } void SimpleMetricsManager::stopThread() { - _runFlag.store(false); + _tickSupplier->kill(); _thread.join(); } diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h index aa2efba49be..693b3f65ec6 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h @@ -57,7 +57,6 @@ private: size_t _maxBuckets; Bucket _totalsBucket; - std::atomic<bool> _runFlag; std::thread _thread; void tickerLoop(); void stopThread(); diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp index c71655ff7d0..31a04e67ac4 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp @@ -21,6 +21,12 @@ SimpleTick::SimpleTick() {} TimeStamp +SimpleTick::first() +{ + return now(); +} + +TimeStamp SimpleTick::next(TimeStamp prev) { std::unique_lock<std::mutex> locker(_lock); @@ -28,8 +34,9 @@ SimpleTick::next(TimeStamp prev) TimeStamp curr = now(); if (curr - prev >= oneSec) { return curr; - } - if (curr <= prev) { + } else if (curr < prev) { + // clock was adjusted backwards + prev = curr; _cond.wait_for(locker, oneSec); } else { _cond.wait_for(locker, oneSec - (curr - prev)); @@ -46,4 +53,10 @@ SimpleTick::kill() _cond.notify_all(); } +bool +SimpleTick::alive() const +{ + return _runFlag; +} + } // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h index 653944ea257..74b9d88d255 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h @@ -9,6 +9,7 @@ namespace vespalib::metrics { +// internal class SimpleTick : public Tick { private: std::mutex _lock; @@ -16,8 +17,10 @@ private: std::condition_variable _cond; public: SimpleTick(); + TimeStamp first() override; TimeStamp next(TimeStamp prev) override; void kill() override; + bool alive() const override; }; } // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h b/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h index ca9db21a2d8..b62e0f2a94e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h @@ -8,6 +8,7 @@ namespace vespalib { +/** metrics-internal utility class */ template <typename T> class StableStore { diff --git a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.cpp b/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.cpp deleted file mode 100644 index e69de29bb2d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.cpp +++ /dev/null diff --git a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.h b/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.h deleted file mode 100644 index e69de29bb2d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/ticker_thread.h +++ /dev/null diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index a7418629f81..16d317551e0 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -1070,9 +1070,8 @@ TwoPhaseUpdateOperationTest::testSafePathConditionParseFailureFailsWithIllegalPa "timestamp 0, timestamp of updated doc: 0) " "ReturnCode(ILLEGAL_PARAMETERS, " "Failed to parse test and set condition: " - "Unexpected token at position 16 " - "('==fran...c') in query 'testdoctype1." - "san==fran...cisco',)"s, + "syntax error, unexpected . at column 24 when " + "parsing selection 'testdoctype1.san==fran...cisco')"s, sender.getLastReply(true)); } @@ -1096,7 +1095,8 @@ TwoPhaseUpdateOperationTest::testSafePathConditonUnknownDocTypeFailsWithIllegalP "timestamp 0, timestamp of updated doc: 0) " "ReturnCode(ILLEGAL_PARAMETERS, " "Failed to parse test and set condition: " - "Document type langbein not found)"s, + "Document type 'langbein' not found at column 1 " + "when parsing selection 'langbein.headerval=1234')"s, sender.getLastReply(true)); } diff --git a/storage/src/tests/distributor/visitoroperationtest.cpp b/storage/src/tests/distributor/visitoroperationtest.cpp index 26f4fb3e784..972ccf41bfe 100644 --- a/storage/src/tests/distributor/visitoroperationtest.cpp +++ b/storage/src/tests/distributor/visitoroperationtest.cpp @@ -454,7 +454,8 @@ VisitorOperationTest::testInvalidOrderDocSelection() CPPUNIT_ASSERT_EQUAL( std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " "ReturnCode(ILLEGAL_PARAMETERS, Failed to parse document select " - "string 'id.order(10,3)=1 and dummy': Document type dummy not found)"), + "string 'id.order(10,3)=1 and dummy': Document type 'dummy' not " + "found at column 22 when parsing selection 'id.order(10,3)=1 and dummy')"), runEmptyVisitor( createVisitorCommand("invalidOrderDoc", id, diff --git a/storage/src/vespa/storage/persistence/fieldvisitor.h b/storage/src/vespa/storage/persistence/fieldvisitor.h index 96b73ca779b..4b47c68e33b 100644 --- a/storage/src/vespa/storage/persistence/fieldvisitor.h +++ b/storage/src/vespa/storage/persistence/fieldvisitor.h @@ -42,7 +42,6 @@ public: void visitArithmeticValueNode(const document::select::ArithmeticValueNode &) override {} void visitFunctionValueNode(const document::select::FunctionValueNode &) override {} void visitIdValueNode(const document::select::IdValueNode &) override {} - void visitSearchColumnValueNode(const document::select::SearchColumnValueNode &) override {} void visitFloatValueNode(const document::select::FloatValueNode &) override {} void visitVariableValueNode(const document::select::VariableValueNode &) override {} void visitIntegerValueNode(const document::select::IntegerValueNode &) override {} diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index b214ef4539a..c19dc7cfd27 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -738,10 +738,11 @@ CommunicationManager::run(framework::ThreadHandle& thread) if (_eventQueue.getNext(msg, 100)) { process(msg); } - for (Protocols::iterator it(_earlierGenerations.begin()); + std::lock_guard<std::mutex> guard(_earlierGenerationsLock); + for (EarlierProtocols::iterator it(_earlierGenerations.begin()); !_earlierGenerations.empty() && ((it->first + TEN_MINUTES) < _component.getClock().getTimeInSeconds()); - _earlierGenerations.begin()) + it = _earlierGenerations.begin()) { _earlierGenerations.erase(it); } @@ -766,8 +767,8 @@ void CommunicationManager::updateMessagebusProtocol( if (_mbus.get()) { framework::SecondTime now(_component.getClock().getTimeInSeconds()); mbus::IProtocol::SP newDocumentProtocol(new documentapi::DocumentProtocol( *_component.getLoadTypes(), repo)); + std::lock_guard<std::mutex> guard(_earlierGenerationsLock); _earlierGenerations.push_back(std::make_pair(now, _mbus->getMessageBus().putProtocol(newDocumentProtocol))); - mbus::IProtocol::SP newStorageProtocol(new mbusprot::StorageProtocol(repo, *_component.getLoadTypes())); _earlierGenerations.push_back(std::make_pair(now, _mbus->getMessageBus().putProtocol(newStorageProtocol))); } diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 89b76583cb5..f4f4aa5a236 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -26,6 +26,7 @@ #include <map> #include <queue> #include <atomic> +#include <mutex> namespace mbus { class RPCMessageBus; @@ -138,8 +139,10 @@ private: PriorityQueue _eventQueue; // XXX: Should perhaps use a configsubscriber and poll from StorageComponent ? std::unique_ptr<config::ConfigFetcher> _configFetcher; - typedef std::vector< std::pair<framework::SecondTime, mbus::IProtocol::SP> > Protocols; - Protocols _earlierGenerations; + using EarlierProtocol = std::pair<framework::SecondTime, mbus::IProtocol::SP>; + using EarlierProtocols = std::vector<EarlierProtocol>; + std::mutex _earlierGenerationsLock; + EarlierProtocols _earlierGenerations; void onOpen() override; void onClose() override; diff --git a/vespabase/conf/default-env.txt.in b/vespabase/conf/default-env.txt.in index 5b144e4c301..38a4d0cded5 100644 --- a/vespabase/conf/default-env.txt.in +++ b/vespabase/conf/default-env.txt.in @@ -1,3 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. fallback VESPA_HOME @CMAKE_INSTALL_PREFIX@ override VESPA_USER vespa +override cloudconfig_server__disable_filedistributor true |