diff options
112 files changed, 2539 insertions, 1515 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 17ee05bff1b..417fd2fa695 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -20,9 +20,7 @@ set(AUTORUN_UNIT_TESTS FALSE CACHE BOOL "If TRUE, tests will be run immediately set(WARN_OPTS "-Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wcomment -Wformat -Wparentheses -Wreturn-type -Wswitch -Wtrigraphs -Wunused -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings") # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 ${WARN_OPTS} -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 -DBOOST_DISABLE_ASSERTS -DWITH_SHIPPED_GEOIP -march=westmere -mtune=intel") - -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${VTAG_DEFINES}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 ${WARN_OPTS} -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 -DBOOST_DISABLE_ASSERTS -march=westmere -mtune=intel") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -Wnon-virtual-dtor -fvisibility-inlines-hidden -fdiagnostics-color=auto") diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java index 465b3a18907..4336da0c254 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -23,24 +24,165 @@ public class DefaultMetricConsumers { public Map<String, MetricsConsumer> getDefaultMetricsConsumers() { Map<String, MetricsConsumer> metricsConsumers = new LinkedHashMap<>(); metricsConsumers.put("yamas", getDefaultYamasConsumer()); - metricsConsumers.put("ymon", getDefaultYmonConsumer()); return metricsConsumers; } - private MetricsConsumer getDefaultYmonConsumer(){ + private MetricsConsumer getDefaultYamasConsumer(){ + List<Metric> metrics = new ArrayList<>(); + + metrics.addAll(addSearchNodeMetrics()); + metrics.addAll(addStorageMetrics()); + metrics.addAll(addDocprocMetrics()); + metrics.addAll(addClusterControllerMetrics()); + metrics.addAll(addQrserverMetrics()); + metrics.addAll(addContainerMetrics()); + metrics.addAll(getConfigServerMetrics()); + + return new MetricsConsumer("yamas", toMapByName(metrics)); + } + + private Map<String, Metric> toMapByName(List<Metric> metrics) { Map<String, Metric> metricMap = new LinkedHashMap<>(); - for (Metric metric : commonMetrics()) { + for (Metric metric : metrics) { metricMap.put(metric.getName(), metric); } + return metricMap; + } + + private Collection<Metric> getConfigServerMetrics() { + List<Metric> metrics = new ArrayList<>(); + + metrics.add(new Metric("configserver.requests.count", "configserver.requests")); + metrics.add(new Metric("configserver.failedRequests.count", "configserver.failedRequests")); + metrics.add(new Metric("configserver.latency.average", "configserver.latency")); + metrics.add(new Metric("configserver.cacheConfigElems.last", "configserver.cacheConfigElems")); + metrics.add(new Metric("configserver.cacheChecksumElems.last", "configserver.cacheChecksumElems")); + metrics.add(new Metric("configserver.hosts.last", "configserver.hosts")); + metrics.add(new Metric("configserver.delayedResponses.count", "configserver.delayedResponses")); + metrics.add(new Metric("configserver.sessionChangeErrors.count", "configserver.sessionChangeErrors")); - return new MetricsConsumer("ymon", metricMap); + return metrics; } - private MetricsConsumer getDefaultYamasConsumer(){ - // include common metrics - List<Metric> metrics = commonMetrics(); + private Collection<Metric> addContainerMetrics() { + List<Metric> metrics = new ArrayList<>(); + + metrics.add(new Metric("serverRejectedRequests.rate")); + metrics.add(new Metric("serverRejectedRequests.count")); + + metrics.add(new Metric("serverThreadPoolSize.average")); + metrics.add(new Metric("serverThreadPoolSize.min")); + metrics.add(new Metric("serverThreadPoolSize.max")); + metrics.add(new Metric("serverThreadPoolSize.rate")); + metrics.add(new Metric("serverThreadPoolSize.count")); + metrics.add(new Metric("serverThreadPoolSize.last")); + + metrics.add(new Metric("serverActiveThreads.average")); + metrics.add(new Metric("serverActiveThreads.min")); + metrics.add(new Metric("serverActiveThreads.max")); + metrics.add(new Metric("serverActiveThreads.rate")); + metrics.add(new Metric("serverActiveThreads.count")); + metrics.add(new Metric("serverActiveThreads.last")); + + metrics.add(new Metric("httpapi_latency.average")); + metrics.add(new Metric("httpapi_pending.average")); + metrics.add(new Metric("httpapi_num_operations.rate")); + metrics.add(new Metric("httpapi_num_updates.rate")); + metrics.add(new Metric("httpapi_num_removes.rate")); + metrics.add(new Metric("httpapi_num_puts.rate")); + metrics.add(new Metric("httpapi_succeeded.rate")); + metrics.add(new Metric("httpapi_failed.rate")); + + metrics.add(new Metric("mem.heap.total.average")); + metrics.add(new Metric("mem.heap.free.average")); + metrics.add(new Metric("mem.heap.used.average")); + + return metrics; + } + + private Collection<Metric> addClusterControllerMetrics() { + List<Metric> metrics = new ArrayList<>(); + + metrics.add(new Metric("cluster-controller.down.count.last")); + metrics.add(new Metric("cluster-controller.initializing.count.last")); + metrics.add(new Metric("cluster-controller.maintenance.count.last")); + metrics.add(new Metric("cluster-controller.retired.count.last")); + metrics.add(new Metric("cluster-controller.stopping.count.last")); + metrics.add(new Metric("cluster-controller.up.count.last")); + metrics.add(new Metric("cluster-controller.cluster-state-change.count", "content.cluster-controller.cluster-state-change.count")); + + metrics.add(new Metric("cluster-controller.is-master.last")); + // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. + // DO NOT RELY ON THIS METRIC YET. + metrics.add(new Metric("cluster-controller.node-event.count")); + + return metrics; + } + + private Collection<Metric> addDocprocMetrics() { + List<Metric> metrics = new ArrayList<>(); + + // per chain + metrics.add(new Metric("documents_processed.rate", "documents_processed")); + + return metrics; + } + + private Collection<Metric> addQrserverMetrics() { + List<Metric> metrics = new ArrayList<>(); + + metrics.add(new Metric("peak_qps.average", "peak_qps")); + metrics.add(new Metric("search_connections.average", "search_connections")); + metrics.add(new Metric("active_queries.average", "active_queries")); + metrics.add(new Metric("queries.rate", "queries")); + metrics.add(new Metric("query_latency.average", "mean_query_latency")); + metrics.add(new Metric("query_latency.max", "max_query_latency")); + metrics.add(new Metric("query_latency.95percentile", "95p_query_latency")); + metrics.add(new Metric("query_latency.99percentile", "99p_query_latency")); + metrics.add(new Metric("failed_queries.rate", "failed_queries")); + metrics.add(new Metric("hits_per_query.average", "hits_per_query")); + metrics.add(new Metric("empty_results.rate", "empty_results")); + metrics.add(new Metric("requestsOverQuota.rate")); + metrics.add(new Metric("requestsOverQuota.count")); + + // Errors from qrserver + metrics.add(new Metric("error.timeout.rate","error.timeout")); + metrics.add(new Metric("error.backends_oos.rate","error.backends_oos")); + metrics.add(new Metric("error.plugin_failure.rate","error.plugin_failure")); + metrics.add(new Metric("error.backend_communication_error.rate","error.backend_communication_error")); + metrics.add(new Metric("error.empty_document_summaries.rate","error.empty_document_summaries")); + metrics.add(new Metric("error.invalid_query_parameter.rate","error.invalid_query_parameter")); + metrics.add(new Metric("error.internal_server_error.rate", "error.internal_server_error")); + metrics.add(new Metric("error.misconfigured_server.rate","error.misconfigured_server")); + metrics.add(new Metric("error.invalid_query_transformation.rate","error.invalid_query_transformation")); + metrics.add(new Metric("error.result_with_errors.rate","error.result_with_errors")); + metrics.add(new Metric("error.unspecified.rate","error.unspecified")); + metrics.add(new Metric("error.unhandled_exception.rate","error.unhandled_exception")); + metrics.add(new Metric("http.status.1xx.rate")); + metrics.add(new Metric("http.status.2xx.rate")); + metrics.add(new Metric("http.status.3xx.rate")); + metrics.add(new Metric("http.status.4xx.rate")); + metrics.add(new Metric("http.status.5xx.rate")); + + return metrics; + } + + private Collection<Metric> addSearchNodeMetrics() { + List<Metric> metrics = new ArrayList<>(); + + metrics.add(new Metric("proton.numstoreddocs.last", "documents_total")); + metrics.add(new Metric("proton.numindexeddocs.last", "documents_ready")); + metrics.add(new Metric("proton.numactivedocs.last", "documents_active")); + metrics.add(new Metric("proton.numremoveddocs.last", "documents_removed")); + + metrics.add(new Metric("proton.docsinmemory.last", "documents_inmemory")); + metrics.add(new Metric("proton.diskusage.last", "diskusage")); + metrics.add(new Metric("proton.memoryusage.max", "content.proton.memoryusage.max")); + metrics.add(new Metric("proton.transport.query.count.rate", "query_requests")); + metrics.add(new Metric("proton.transport.docsum.docs.rate", "document_requests")); + metrics.add(new Metric("proton.transport.docsum.latency.average", "content.proton.transport.docsum.latency.average")); + metrics.add(new Metric("proton.transport.query.latency.average", "query_latency")); - //Search node // jobs metrics.add(new Metric("content.proton.documentdb.job.total.average")); metrics.add(new Metric("content.proton.documentdb.job.attribute_flush.average")); @@ -83,8 +225,23 @@ public class DefaultMetricConsumers { metrics.add(new Metric("content.proton.documentdb.removed.document_store.disk_bloat.average")); metrics.add(new Metric("content.proton.documentdb.removed.document_store.max_bucket_spread.average")); + return metrics; + } + + private Collection<Metric> addStorageMetrics() { + List<Metric> metrics = new ArrayList<>(); + + metrics.add(new Metric("vds.datastored.alldisks.docs.average","docs")); + metrics.add(new Metric("vds.datastored.alldisks.bytes.average","bytes")); + metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average","visitorlifetime")); + metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average","visitorqueuewait")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate","put")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate","remove")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate","get")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate","update")); + metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize")); + metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait")); - //Storage metrics.add(new Metric("vds.memfilepersistence.cache.files.average")); metrics.add(new Metric("vds.memfilepersistence.cache.body.average")); metrics.add(new Metric("vds.memfilepersistence.cache.header.average")); @@ -116,13 +273,25 @@ public class DefaultMetricConsumers { //Distributor metrics.add(new Metric("vds.visitor.sum.latency.average")); metrics.add(new Metric("vds.visitor.sum.failed.rate")); + metrics.add(new Metric("vds.idealstate.buckets_rechecking.average")); metrics.add(new Metric("vds.idealstate.idealstate_diff.average")); metrics.add(new Metric("vds.idealstate.buckets_toofewcopies.average")); metrics.add(new Metric("vds.idealstate.buckets_toomanycopies.average")); metrics.add(new Metric("vds.idealstate.buckets.average")); metrics.add(new Metric("vds.idealstate.buckets_notrusted.average")); - + metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate","deleteok")); + metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate","deletefailed")); + metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average","deletepending")); + metrics.add(new Metric("vds.idealstate.merge_bucket.done_ok.rate","mergeok")); + metrics.add(new Metric("vds.idealstate.merge_bucket.done_failed.rate","mergefailed")); + metrics.add(new Metric("vds.idealstate.merge_bucket.pending.average","mergepending")); + metrics.add(new Metric("vds.idealstate.split_bucket.done_ok.rate","splitok")); + metrics.add(new Metric("vds.idealstate.split_bucket.done_failed.rate","splitfailed")); + metrics.add(new Metric("vds.idealstate.split_bucket.pending.average","splitpending")); + metrics.add(new Metric("vds.idealstate.join_bucket.done_ok.rate","joinok")); + metrics.add(new Metric("vds.idealstate.join_bucket.done_failed.rate","joinfailed")); + metrics.add(new Metric("vds.idealstate.join_bucket.pending.average","joinpending")); metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); metrics.add(new Metric("vds.distributor.puts.sum.ok.rate")); @@ -141,156 +310,6 @@ public class DefaultMetricConsumers { metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate")); metrics.add(new Metric("vds.distributor.docsstored.average")); metrics.add(new Metric("vds.distributor.bytesstored.average")); - metrics.add(new Metric("vds.visitor.sum.latency.average")); - metrics.add(new Metric("vds.visitor.sum.failed.rate")); - - // Cluster Controller - metrics.add(new Metric("cluster-controller.down.count.last")); - metrics.add(new Metric("cluster-controller.initializing.count.last")); - metrics.add(new Metric("cluster-controller.maintenance.count.last")); - metrics.add(new Metric("cluster-controller.retired.count.last")); - metrics.add(new Metric("cluster-controller.stopping.count.last")); - metrics.add(new Metric("cluster-controller.up.count.last")); - metrics.add(new Metric("cluster-controller.cluster-state-change.count", "content.cluster-controller.cluster-state-change.count")); - - metrics.add(new Metric("cluster-controller.is-master.last")); - // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. - // DO NOT RELY ON THIS METRIC YET. - metrics.add(new Metric("cluster-controller.node-event.count")); - - // Errors from qrserver - metrics.add(new Metric("error.timeout.rate","error.timeout")); - metrics.add(new Metric("error.backends_oos.rate","error.backends_oos")); - metrics.add(new Metric("error.plugin_failure.rate","error.plugin_failure")); - metrics.add(new Metric("error.backend_communication_error.rate","error.backend_communication_error")); - metrics.add(new Metric("error.empty_document_summaries.rate","error.empty_document_summaries")); - metrics.add(new Metric("error.invalid_query_parameter.rate","error.invalid_query_parameter")); - metrics.add(new Metric("error.internal_server_error.rate", "error.internal_server_error")); - metrics.add(new Metric("error.misconfigured_server.rate","error.misconfigured_server")); - metrics.add(new Metric("error.invalid_query_transformation.rate","error.invalid_query_transformation")); - metrics.add(new Metric("error.result_with_errors.rate","error.result_with_errors")); - metrics.add(new Metric("error.unspecified.rate","error.unspecified")); - metrics.add(new Metric("error.unhandled_exception.rate","error.unhandled_exception")); - metrics.add(new Metric("http.status.1xx.rate")); - metrics.add(new Metric("http.status.2xx.rate")); - metrics.add(new Metric("http.status.3xx.rate")); - metrics.add(new Metric("http.status.4xx.rate")); - metrics.add(new Metric("http.status.5xx.rate")); - - - // container - metrics.add(new Metric("serverRejectedRequests.rate")); - metrics.add(new Metric("serverRejectedRequests.count")); - - metrics.add(new Metric("serverThreadPoolSize.average")); - metrics.add(new Metric("serverThreadPoolSize.min")); - metrics.add(new Metric("serverThreadPoolSize.max")); - metrics.add(new Metric("serverThreadPoolSize.rate")); - metrics.add(new Metric("serverThreadPoolSize.count")); - metrics.add(new Metric("serverThreadPoolSize.last")); - - metrics.add(new Metric("serverActiveThreads.average")); - metrics.add(new Metric("serverActiveThreads.min")); - metrics.add(new Metric("serverActiveThreads.max")); - metrics.add(new Metric("serverActiveThreads.rate")); - metrics.add(new Metric("serverActiveThreads.count")); - metrics.add(new Metric("serverActiveThreads.last")); - - metrics.add(new Metric("httpapi_latency.average")); - metrics.add(new Metric("httpapi_pending.average")); - metrics.add(new Metric("httpapi_num_operations.rate")); - metrics.add(new Metric("httpapi_num_updates.rate")); - metrics.add(new Metric("httpapi_num_removes.rate")); - metrics.add(new Metric("httpapi_num_puts.rate")); - metrics.add(new Metric("httpapi_succeeded.rate")); - metrics.add(new Metric("httpapi_failed.rate")); - - metrics.add(new Metric("mem.heap.total.average")); - metrics.add(new Metric("mem.heap.free.average")); - metrics.add(new Metric("mem.heap.used.average")); - - - // Config server - metrics.add(new Metric("configserver.requests.count", "configserver.requests")); - metrics.add(new Metric("configserver.failedRequests.count", "configserver.failedRequests")); - metrics.add(new Metric("configserver.latency.average", "configserver.latency")); - metrics.add(new Metric("configserver.cacheConfigElems.last", "configserver.cacheConfigElems")); - metrics.add(new Metric("configserver.cacheChecksumElems.last", "configserver.cacheChecksumElems")); - metrics.add(new Metric("configserver.hosts.last", "configserver.hosts")); - metrics.add(new Metric("configserver.delayedResponses.count", "configserver.delayedResponses")); - metrics.add(new Metric("configserver.sessionChangeErrors.count", "configserver.sessionChangeErrors")); - - - Map<String, Metric> metricMap = new LinkedHashMap<>(); - for (Metric metric : metrics) { - metricMap.put(metric.getName(), metric); - } - - return new MetricsConsumer("yamas", metricMap); - } - - // Common metrics for ymon and yamas. For ymon metric names needs to be less then 19 characters long - private List<Metric> commonMetrics(){ - List<Metric> metrics = new ArrayList<>(); - - //Searchnode - metrics.add(new Metric("proton.numstoreddocs.last", "documents_total")); - metrics.add(new Metric("proton.numindexeddocs.last", "documents_ready")); - metrics.add(new Metric("proton.numactivedocs.last", "documents_active")); - metrics.add(new Metric("proton.numremoveddocs.last", "documents_removed")); - - metrics.add(new Metric("proton.docsinmemory.last", "documents_inmemory")); - metrics.add(new Metric("proton.diskusage.last", "diskusage")); - metrics.add(new Metric("proton.memoryusage.max", "content.proton.memoryusage.max")); - metrics.add(new Metric("proton.transport.query.count.rate", "query_requests")); - metrics.add(new Metric("proton.transport.docsum.docs.rate", "document_requests")); - metrics.add(new Metric("proton.transport.docsum.latency.average", "content.proton.transport.docsum.latency.average")); - metrics.add(new Metric("proton.transport.query.latency.average", "query_latency")); - - //Docproc - per chain - metrics.add(new Metric("documents_processed.rate", "documents_processed")); - - //Qrserver - metrics.add(new Metric("peak_qps.average", "peak_qps")); - metrics.add(new Metric("search_connections.average", "search_connections")); - metrics.add(new Metric("active_queries.average", "active_queries")); - metrics.add(new Metric("queries.rate", "queries")); - metrics.add(new Metric("query_latency.average", "mean_query_latency")); - metrics.add(new Metric("query_latency.max", "max_query_latency")); - metrics.add(new Metric("query_latency.95percentile", "95p_query_latency")); - metrics.add(new Metric("query_latency.99percentile", "99p_query_latency")); - metrics.add(new Metric("failed_queries.rate", "failed_queries")); - metrics.add(new Metric("hits_per_query.average", "hits_per_query")); - metrics.add(new Metric("empty_results.rate", "empty_results")); - metrics.add(new Metric("requestsOverQuota.rate")); - metrics.add(new Metric("requestsOverQuota.count")); - - //Storage - metrics.add(new Metric("vds.datastored.alldisks.docs.average","docs")); - metrics.add(new Metric("vds.datastored.alldisks.bytes.average","bytes")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average","visitorlifetime")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average","visitorqueuewait")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate","put")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate","remove")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate","get")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate","update")); - metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize")); - metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait")); - - - //Distributor - metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate","deleteok")); - metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate","deletefailed")); - metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average","deletepending")); - metrics.add(new Metric("vds.idealstate.merge_bucket.done_ok.rate","mergeok")); - metrics.add(new Metric("vds.idealstate.merge_bucket.done_failed.rate","mergefailed")); - metrics.add(new Metric("vds.idealstate.merge_bucket.pending.average","mergepending")); - metrics.add(new Metric("vds.idealstate.split_bucket.done_ok.rate","splitok")); - metrics.add(new Metric("vds.idealstate.split_bucket.done_failed.rate","splitfailed")); - metrics.add(new Metric("vds.idealstate.split_bucket.pending.average","splitpending")); - metrics.add(new Metric("vds.idealstate.join_bucket.done_ok.rate","joinok")); - metrics.add(new Metric("vds.idealstate.join_bucket.done_failed.rate","joinfailed")); - metrics.add(new Metric("vds.idealstate.join_bucket.pending.average","joinpending")); return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index c1ad6eead47..91f14eb76fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -56,11 +56,11 @@ public class ContainerDocumentApi implements FeederConfig.Producer { Set<ComponentSpecification> inherited = new TreeSet<>(); SearchChain vespaGetChain = new SearchChain(new ChainSpecification(new ComponentId("vespaget"), - new ChainSpecification.Inheritance(inherited, null), new ArrayList<Phase>(), new TreeSet<ComponentSpecification>())); + new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); vespaGetChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.GetSearcher")); SearchChain vespaVisitChain = new SearchChain(new ChainSpecification(new ComponentId("vespavisit"), - new ChainSpecification.Inheritance(inherited, null), new ArrayList<Phase>(), new TreeSet<ComponentSpecification>())); + new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); vespaVisitChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.VisitSearcher")); SearchChains chains; @@ -76,8 +76,8 @@ public class ContainerDocumentApi implements FeederConfig.Producer { ContainerSearch containerSearch = new ContainerSearch(cluster, chains, new ContainerSearch.Options()); cluster.setSearch(containerSearch); - final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( - chains, "com.yahoo.search.handler.SearchHandler"); + ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(chains, + "com.yahoo.search.handler.SearchHandler"); searchHandler.addServerBindings("http://*/search/*", "https://*/search/*"); cluster.addComponent(searchHandler); } @@ -162,4 +162,5 @@ public class ContainerDocumentApi implements FeederConfig.Producer { this.docprocChain = docprocChain; } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 7b9ca803719..7b251713225 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -17,8 +17,12 @@ public class DomResourceLimitsBuilder { if (resourceLimits == null) { return builder.build(); } - builder.setDiskLimit(resourceLimits.childAsDouble("disk")); - builder.setMemoryLimit(resourceLimits.childAsDouble("memory")); + if (resourceLimits.getChild("disk") != null) { + builder.setDiskLimit(resourceLimits.childAsDouble("disk")); + } + if (resourceLimits.getChild("memory") != null) { + builder.setMemoryLimit(resourceLimits.childAsDouble("memory")); + } return builder.build(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 9d82e663ea6..c05477921f3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -39,6 +39,12 @@ public class ContentSearchClusterTest { return new ProtonConfig(protonCfgBuilder); } + private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimits, String clusterXml) throws Exception { + ProtonConfig cfg = getProtonConfig(createCluster(clusterXml)); + assertEquals(expDiskLimit, cfg.writefilter().disklimit(), EPSILON); + assertEquals(expMemoryLimits, cfg.writefilter().memorylimit(), EPSILON); + } + @Test public void requireThatProtonInitializeThreadsIsSet() throws Exception { assertEquals(2, getProtonConfig(createClusterWithOneDocumentType()).initialize().threads()); @@ -47,9 +53,20 @@ public class ContentSearchClusterTest { @Test public void requireThatProtonResourceLimitsCanBeSet() throws Exception { - String clusterXml = new ContentClusterBuilder().protonDiskLimit(0.88).protonMemoryLimit(0.77).getXml(); - ProtonConfig cfg = getProtonConfig(createCluster(clusterXml)); - assertEquals(0.88, cfg.writefilter().disklimit(), EPSILON); - assertEquals(0.77, cfg.writefilter().memorylimit(), EPSILON); + assertProtonResourceLimits(0.88, 0.77, + new ContentClusterBuilder().protonDiskLimit(0.88).protonMemoryLimit(0.77).getXml()); } + + @Test + public void requireThatOnlyDiskLimitCanBeSet() throws Exception { + assertProtonResourceLimits(0.88, 0.9, + new ContentClusterBuilder().protonDiskLimit(0.88).getXml()); + } + + @Test + public void requireThatOnlyMemoryLimitCanBeSet() throws Exception { + assertProtonResourceLimits(0.9, 0.77, + new ContentClusterBuilder().protonMemoryLimit(0.77).getXml()); + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 35666832929..6a397d0e17a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -70,7 +70,19 @@ public final class ClusterSpec { return true; } + /** Returns whether this is equal, disregarding the group value and wanted docker image */ + public boolean equalsIgnoringGroupAndDockerImage(Object o) { + if (o == this) return true; + if ( ! (o instanceof ClusterSpec)) return false; + ClusterSpec other = (ClusterSpec)o; + if ( ! other.type.equals(this.type)) return false; + if ( ! other.id.equals(this.id)) return false; + return true; + } + + // TODO: Remove when no version older than 6.41 is used /** Returns whether this is equal, disregarding the group value */ + @Deprecated public boolean equalsIgnoringGroup(Object o) { if (o == this) return true; if ( ! (o instanceof ClusterSpec)) return false; diff --git a/config/src/vespa/config/common/vespa_version.cpp b/config/src/vespa/config/common/vespa_version.cpp index 595f4dfd23b..5e23d5889e6 100644 --- a/config/src/vespa/config/common/vespa_version.cpp +++ b/config/src/vespa/config/common/vespa_version.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "vespa_version.h" - -#ifndef V_TAG_COMPONENT -#define V_TAG_COMPONENT "1.0.0" -#endif +#include <vespa/vespalib/component/vtag.h> namespace config { -const VespaVersion currentVersion(VespaVersion::fromString(vespalib::string(V_TAG_COMPONENT))); +const VespaVersion currentVersion(VespaVersion::fromString(vespalib::string(vespalib::VersionTagComponent))); VespaVersion::VespaVersion(const VespaVersion & vespaVersion) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java index ec681bf0a70..2850241f493 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java @@ -36,7 +36,7 @@ public class StaticProvisioner implements HostProvisioner { if (requestedCluster.group().isPresent()) // we are requesting a specific group return nodeCluster.equals(requestedCluster); else // we are requesting nodes of all groups in this cluster - return nodeCluster.equalsIgnoringGroup(requestedCluster); + return nodeCluster.equalsIgnoringGroupAndDockerImage(requestedCluster); } } diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 9b61e60d759..c180780faf9 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -1,5 +1,6 @@ <?xml version="1.0"?> <!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- This module collects all common dependencies of Vespa-internal modules, i.e the non-leaf container modules --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 36ae057b418..2f431b7799e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -341,10 +341,10 @@ public class ClusterSearcher extends Searcher { if (invalidInDocTypes != null && !invalidInDocTypes.isEmpty()) { String plural = invalidInDocTypes.size() > 1 ? "s" : ""; - return new Result(query, ErrorMessage.createInvalidQueryParameter( - "Requested rank profile '" + rankProfile + - "' is undefined for document type" + plural + " '" + - StringUtils.join(invalidInDocTypes.iterator(), ", ") + "'")); + return new Result(query, + ErrorMessage.createInvalidQueryParameter("Requested rank profile '" + rankProfile + + "' is undefined for document type" + plural + " '" + + StringUtils.join(invalidInDocTypes.iterator(), ", ") + "'")); } return null; @@ -373,9 +373,7 @@ public class ClusterSearcher extends Searcher { } } // no error: good result, let's return - if (result.hits().getError() == null) { - return; - } + if (result.hits().getError() == null) return; } while (tries < hasher.getNodeCount() && failoverToRemote); } @@ -387,11 +385,8 @@ public class ClusterSearcher extends Searcher { private void updateCacheHitRatio(Result result, Query query) { // result.isCached() looks at the contained hits, so if there are no - // hits, the result will be treated as cached, even though the backend - // was queried. - if (result.hits().getError() == null - && result.hits().getConcreteSize() > 0) { - + // hits, the result will be treated as cached, even though the backend was queried. + if (result.hits().getError() == null && result.hits().getConcreteSize() > 0) { if (result.isCached()) { cacheHit(); } else if (!query.getNoCache()) { @@ -401,7 +396,7 @@ public class ClusterSearcher extends Searcher { } @Override - public Result search(com.yahoo.search.Query query, Execution execution) { + public Result search(Query query, Execution execution) { Result result; int tries = 0; @@ -436,19 +431,23 @@ public class ClusterSearcher extends Searcher { } private void validateQueryTimeout(Query query) { - if (query.getTimeout() > maxQueryTimeout) { - log.warning("Query timeout (" + query.getTimeout() + " ms) > max query timeout (" + maxQueryTimeout + " ms) for '" + - query.toString() + "'. Setting timeout to " + maxQueryTimeout + " ms."); - query.setTimeout(maxQueryTimeout); + if (query.getTimeout() <= maxQueryTimeout) return; + + if (query.isTraceable(2)) { + query.trace("Query timeout (" + query.getTimeout() + " ms) > max query timeout (" + + maxQueryTimeout + " ms). Setting timeout to " + maxQueryTimeout + " ms.", 2); } + query.setTimeout(maxQueryTimeout); } private void validateQueryCache(Query query) { - if (query.getRanking().getQueryCache() && query.getTimeout() > maxQueryCacheTimeout) { - log.warning("Query timeout (" + query.getTimeout() + " ms) > max query cache timeout (" + maxQueryCacheTimeout + " ms) for '" + - query.toString() + "'. Disabling query cache."); - query.getRanking().setQueryCache(false); + if (query.getRanking().getQueryCache() && query.getTimeout() <= maxQueryCacheTimeout) return; + + if (query.isTraceable(2)) { + query.trace("Query timeout (" + query.getTimeout() + " ms) > max query cache timeout (" + + maxQueryCacheTimeout + " ms). Disabling query cache.", 2); } + query.getRanking().setQueryCache(false); } private Result doSearch(Searcher searcher, Query query, Execution execution) { diff --git a/container/OWNERS b/container/OWNERS new file mode 100644 index 00000000000..3b2ba1ede81 --- /dev/null +++ b/container/OWNERS @@ -0,0 +1 @@ +gjoranv diff --git a/container/pom.xml b/container/pom.xml new file mode 100644 index 00000000000..e46b4a4d419 --- /dev/null +++ b/container/pom.xml @@ -0,0 +1,35 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- This module collects all dependencies applications need to create container components. --> +<!-- It should be considered an external Vespa API. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <artifactId>container</artifactId> + <version>6-SNAPSHOT</version> + <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespaclient-container-plugin</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 6f9edc1e106..59d7b987093 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -32,6 +32,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { private Optional<String> networkMode = Optional.empty(); private Optional<String> ipv4Address = Optional.empty(); private Optional<String> ipv6Address = Optional.empty(); + private Optional<String[]> entrypoint = Optional.empty(); CreateContainerCommandImpl(DockerClient docker, DockerImage dockerImage, @@ -56,6 +57,12 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { return this; } + @Override + public Docker.CreateContainerCommand withEntrypoint(String... entrypoint) { + this.entrypoint = Optional.of(entrypoint); + return this; + } + @Override public Docker.CreateContainerCommand withEnvironment(String name, String value) { @@ -119,6 +126,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { if (networkMode.isPresent()) containerCmd = containerCmd.withNetworkMode(networkMode.get()); if (ipv4Address.isPresent()) containerCmd = containerCmd.withIpv4Address(ipv4Address.get()); if (ipv6Address.isPresent()) containerCmd = containerCmd.withIpv6Address(ipv6Address.get()); + if (entrypoint.isPresent()) containerCmd = containerCmd.withEntrypoint(entrypoint.get()); return containerCmd; } @@ -126,7 +134,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { /** Maps ("--env", {"A", "B", "C"}) to "--env A --env B --env C ". */ private String toRepeatedOption(String option, List<String> optionValues) { StringBuilder builder = new StringBuilder(); - optionValues.stream().forEach(optionValue -> builder.append(option).append(" ").append(optionValue).append(" ")); + optionValues.forEach(optionValue -> builder.append(option).append(" ").append(optionValue).append(" ")); return builder.toString(); } @@ -153,6 +161,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { + toOptionalOption("--net", networkMode) + toOptionalOption("--ip", ipv4Address) + toOptionalOption("--ip6", ipv6Address) + + toOptionalOption("--entrypoint", entrypoint) + dockerImage.asString(); } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 5720b18ac57..c7ef19ccb2a 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi; +import java.io.File; import java.net.InetAddress; import java.util.List; import java.util.Map; @@ -21,6 +22,7 @@ public interface Docker { CreateContainerCommand withNetworkMode(String mode); CreateContainerCommand withIpAddress(InetAddress address); CreateContainerCommand withUlimit(String name, int softLimit, int hardLimit); + CreateContainerCommand withEntrypoint(String... entrypoint); void create(); } @@ -68,6 +70,8 @@ public interface Docker { void deleteImage(DockerImage dockerImage); + void buildImage(File dockerfile, DockerImage dockerImage); + /** * Deletes the local images that are currently not in use by any container and not in the except set. */ diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 464c79be874..80bcbeb6537 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -15,6 +15,7 @@ import com.github.dockerjava.core.DefaultDockerClientConfig; import com.github.dockerjava.core.DockerClientImpl; import com.github.dockerjava.core.RemoteApiVersion; import com.github.dockerjava.core.async.ResultCallbackTemplate; +import com.github.dockerjava.core.command.BuildImageResultCallback; import com.github.dockerjava.core.command.ExecStartResultCallback; import com.github.dockerjava.core.command.PullImageResultCallback; import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory; @@ -28,6 +29,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import javax.annotation.concurrent.GuardedBy; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.net.Inet6Address; import java.net.InetAddress; @@ -254,11 +256,9 @@ public class DockerImpl implements Docker { return new ContainerStatsImpl(statsCallback.stats.getNetworks(), statsCallback.stats.getCpuStats(), statsCallback.stats.getMemoryStats(), statsCallback.stats.getBlkioStats()); - } catch (DockerException e) { + } catch (DockerException | InterruptedException e) { numberOfDockerDaemonFails.add(); throw new RuntimeException("Failed to get container stats", e); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to get container stats in time", e); } } @@ -379,6 +379,17 @@ public class DockerImpl implements Docker { } } + @Override + public void buildImage(File dockerfile, DockerImage image) { + try { + dockerClient.buildImageCmd(dockerfile).withTag(image.asString()) + .exec(new BuildImageResultCallback()).awaitCompletion(); + } catch (DockerException | InterruptedException e) { + numberOfDockerDaemonFails.add(); + throw new RuntimeException("Failed to build image " + image.asString(), e); + } + } + private Map<String, Image> filterOutImagesUsedByContainers( Map<String, Image> dockerImagesByImageId, List<com.github.dockerjava.api.model.Container> containerList) { Map<String, Image> filteredDockerImagesByImageId = new HashMap<>(dockerImagesByImageId); @@ -458,7 +469,7 @@ public class DockerImpl implements Docker { @Override public void deleteUnusedDockerImages(Set<DockerImage> except) { - getUnusedDockerImages(except).stream().forEach(this::deleteImage); + getUnusedDockerImages(except).forEach(this::deleteImage); } private class ImagePullCallback extends PullImageResultCallback { diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java index fadfbc2d9bd..9af564ef23a 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.dockerapi; import com.github.dockerjava.api.model.Network; -import com.github.dockerjava.core.command.BuildImageResultCallback; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; @@ -62,9 +61,9 @@ public class DockerTestUtils { .withName(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withDriver("bridge").withIpam(ipam).exec(); } - public static void createDockerImage(DockerImpl docker, DockerImage dockerImage) throws IOException, ExecutionException, InterruptedException { + public static void buildSimpleHttpServerDockerImage(DockerImpl docker, DockerImage dockerImage) throws IOException, ExecutionException, InterruptedException { try { - docker.deleteImage(new DockerImage(dockerImage.asString())); + docker.deleteImage(dockerImage); } catch (Exception e) { if (! e.getMessage().equals("Failed to delete docker image " + dockerImage.asString())) { throw e; @@ -72,10 +71,8 @@ public class DockerTestUtils { } // Build the image locally - File dockerFilePath = new File("src/test/resources/simple-ipv6-server"); - docker.dockerClient - .buildImageCmd(dockerFilePath) - .withTag(dockerImage.asString()).exec(new BuildImageResultCallback()).awaitCompletion(); + File dockerFileStream = new File("src/test/resources/simple-ipv6-server"); + docker.buildImage(dockerFileStream, dockerImage); } private enum OS { Linux, Mac_OS_X, Unsupported } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java index 26cbaff720c..842ad722e5b 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java @@ -100,11 +100,17 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim } public String toSecretAgentReport() throws JsonProcessingException { + final Map<String, Object> routing = new HashMap<>(); + final Map<String, Object> routingYamas = new HashMap<>(); + routing.put("yamas", routingYamas); + routingYamas.put("namespaces", new String[]{"Vespa"}); + Map<String, Object> report = new LinkedHashMap<>(); report.put("application", "docker"); report.put("timestamp", System.currentTimeMillis() / 1000); report.put("dimensions", dimensions.dimensionsMap); report.put("metrics", metrics); + report.put("routing", routing); return objectMapper.writeValueAsString(report); } diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java index 0ad369fb84c..a4e1b775919 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java @@ -159,8 +159,7 @@ public class DockerTest { assumeTrue(DockerTestUtils.dockerDaemonIsPresent()); docker = DockerTestUtils.getDocker(); - DockerTestUtils.createDockerTestNetworkIfNeeded(docker); - DockerTestUtils.createDockerImage(docker, dockerImage); + DockerTestUtils.buildSimpleHttpServerDockerImage(docker, dockerImage); } // Clean up any non deleted containers from previous tests diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt index 5e5eb6cf139..efa1bcb0c13 100644 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -21,7 +21,6 @@ vespa_add_library(fastos unix_socket.cpp unix_thread.cpp unix_time.cpp - vtag.cpp INSTALL lib64 DEPENDS ${CMAKE_DL_LIBS} diff --git a/fastos/src/vespa/fastos/vtag.cpp b/fastos/src/vespa/fastos/vtag.cpp deleted file mode 100644 index a9f00820224..00000000000 --- a/fastos/src/vespa/fastos/vtag.cpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdio.h> -#include "vtag.h" - -#ifndef V_TAG -#define V_TAG "NOTAG" -#define V_TAG_TYPE "NOTAG" -#define V_TAG_VALUE "NOTAG" -#define V_TAG_DATE "NOTAG" -#define V_TAG_SYSTEM "NOTAG" -#define V_TAG_SYSTEM_REV "NOTAG" -#define V_TAG_BUILDER "NOTAG" -#endif - -namespace fastos { - -char VersionTag[] = V_TAG; -char VersionTagDate[] = V_TAG_DATE; -char VersionTagSystem[] = V_TAG_SYSTEM; -char VersionTagSystemRev[] = V_TAG_SYSTEM_REV; -char VersionTagBuilder[] = V_TAG_BUILDER; - -} // namespace fastos diff --git a/fastos/src/vespa/fastos/vtag.h b/fastos/src/vespa/fastos/vtag.h deleted file mode 100644 index ee5c065ef1c..00000000000 --- a/fastos/src/vespa/fastos/vtag.h +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace fastos { - -extern char VersionTag[]; -extern char VersionTagType[]; -extern char VersionTagValue[]; -extern char VersionTagDate[]; -extern char VersionTagSystem[]; -extern char VersionTagSystemRev[]; -extern char VersionTagBuilder[]; - -} // namespace fastos - diff --git a/fnet/src/vespa/fnet/CMakeLists.txt b/fnet/src/vespa/fnet/CMakeLists.txt index 2df853b2893..556cb69a988 100644 --- a/fnet/src/vespa/fnet/CMakeLists.txt +++ b/fnet/src/vespa/fnet/CMakeLists.txt @@ -22,7 +22,6 @@ vespa_add_library(fnet task.cpp transport.cpp transport_thread.cpp - vtag.cpp $<TARGET_OBJECTS:fnet_frt> INSTALL lib64 DEPENDS diff --git a/fnet/src/vespa/fnet/fnet.h b/fnet/src/vespa/fnet/fnet.h index b4ca0d95599..85f4f3e750e 100644 --- a/fnet/src/vespa/fnet/fnet.h +++ b/fnet/src/vespa/fnet/fnet.h @@ -3,15 +3,12 @@ #pragma once #include <vespa/fastos/fastos.h> +#include <vespa/vespalib/component/vtag.h> // FEATURES #include "features.h" -// VTAG - -#include "vtag.h" - // DEFINES #define FNET_NOID ((uint32_t)-1) diff --git a/fnet/src/vespa/fnet/info.cpp b/fnet/src/vespa/fnet/info.cpp index c345fc2e28c..1ad3dbb784a 100644 --- a/fnet/src/vespa/fnet/info.cpp +++ b/fnet/src/vespa/fnet/info.cpp @@ -61,7 +61,7 @@ FNET_Info::FNET_Info() const char* FNET_Info::GetFNETVersion() { - return fnet::VersionTag; + return vespalib::VersionTag; } diff --git a/fnet/src/vespa/fnet/vtag.cpp b/fnet/src/vespa/fnet/vtag.cpp deleted file mode 100644 index e2f280dee5b..00000000000 --- a/fnet/src/vespa/fnet/vtag.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/fnet/fnet.h> - -#ifndef V_TAG -#define V_TAG "NOTAG" -#define V_TAG_DATE "NOTAG" -#define V_TAG_SYSTEM "NOTAG" -#define V_TAG_SYSTEM_REV "NOTAG" -#define V_TAG_BUILDER "NOTAG" -#endif - -namespace fnet { - -char VersionTag[] = V_TAG; -char VersionTagDate[] = V_TAG_DATE; -char VersionTagSystem[] = V_TAG_SYSTEM; -char VersionTagSystemRev[] = V_TAG_SYSTEM_REV; -char VersionTagBuilder[] = V_TAG_BUILDER; - -} // namespace fnet diff --git a/fnet/src/vespa/fnet/vtag.h b/fnet/src/vespa/fnet/vtag.h deleted file mode 100644 index 28b3164e9fc..00000000000 --- a/fnet/src/vespa/fnet/vtag.h +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace fnet { - -extern char VersionTag[]; -extern char VersionTagDate[]; -extern char VersionTagSystem[]; -extern char VersionTagSystemRev[]; -extern char VersionTagBuilder[]; - -} // namespace fnet - diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index 3c0f7bacd33..81aa64641b2 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_add_library(logd STATIC cmdbuf.cpp perform.cpp sigterm.cpp - vtag.cpp DEPENDS ) vespa_generate_config(logd ../main/resources/configdefinitions/logd.def) diff --git a/logd/src/logd/forward.cpp b/logd/src/logd/forward.cpp index 373f95ab742..e4c86583fc0 100644 --- a/logd/src/logd/forward.cpp +++ b/logd/src/logd/forward.cpp @@ -7,15 +7,14 @@ #include <unistd.h> #include <time.h> #include <assert.h> - -#include <vespa/log/log.h> -LOG_SETUP(""); -LOG_RCSID("$Id$"); - #include "errhandle.h" #include "service.h" #include "forward.h" -#include "vtag.h" +#include <vespa/vespalib/component/vtag.h> +#include <vespa/log/log.h> + +LOG_SETUP(""); +LOG_RCSID("$Id$"); namespace logdemon { @@ -40,7 +39,7 @@ void Forwarder::sendMode() { char buf[1024]; - snprintf(buf, 1024, "mode logd %s\n", VersionTag); + snprintf(buf, 1024, "mode logd %s\n", vespalib::VersionTag); int len = strlen(buf); if (len < 100) { forwardText(buf, len); diff --git a/logd/src/logd/service.h b/logd/src/logd/service.h index a6d6ca4ff60..700ab85fe27 100644 --- a/logd/src/logd/service.h +++ b/logd/src/logd/service.h @@ -3,6 +3,7 @@ #include <assert.h> #include <logd/config-logd.h> #include <vespa/vespalib/util/hashmap.h> +#include <vespa/log/log.h> namespace logdemon { diff --git a/logd/src/logd/vtag.cpp b/logd/src/logd/vtag.cpp deleted file mode 100644 index 17b677d4662..00000000000 --- a/logd/src/logd/vtag.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace logdemon { - -#ifndef V_TAG -#define V_TAG "NOTAG" -#define V_TAG_DATE "NOTAG" -#define V_TAG_SYSTEM "NOTAG" -#define V_TAG_SYSTEM_REV "NOTAG" -#define V_TAG_BUILDER "NOTAG" -#endif - -char VersionTag[] = V_TAG; -char VersionTagDate[] = V_TAG_DATE; -char VersionTagSystem[] = V_TAG_SYSTEM; -char VersionTagSystemRev[] = V_TAG_SYSTEM_REV; -char VersionTagBuilder[] = V_TAG_BUILDER; - -} // namespace logdemon diff --git a/logd/src/logd/vtag.h b/logd/src/logd/vtag.h deleted file mode 100644 index 7c78914829e..00000000000 --- a/logd/src/logd/vtag.h +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace logdemon { - -extern char VersionTag[]; -extern char VersionTagDate[]; -extern char VersionTagSystem[]; -extern char VersionTagSystemRev[]; -extern char VersionTagBuilder[]; - -} - diff --git a/messagebus/src/apps/printversion/printversion.cpp b/messagebus/src/apps/printversion/printversion.cpp index 8401653fc51..d616394ef80 100644 --- a/messagebus/src/apps/printversion/printversion.cpp +++ b/messagebus/src/apps/printversion/printversion.cpp @@ -1,19 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/messagebus/vtag.h> +#include <vespa/vespalib/component/vtag.h> #include <stdio.h> -#include <vespa/vespalib/component/version.h> int main(int, char **) { - printf("version tag: %s\n", mbus::VersionTag); - printf("version tag date: %s\n", mbus::VersionTagDate); - printf("version tag system: %s\n", mbus::VersionTagSystem); - printf("version tag system rev: %s\n", mbus::VersionTagSystemRev); - printf("version tag builder: %s\n", mbus::VersionTagBuilder); + printf("version tag: %s\n", vespalib::VersionTag); + printf("version tag date: %s\n", vespalib::VersionTagDate); + printf("version tag system: %s\n", vespalib::VersionTagSystem); + printf("version tag system rev: %s\n", vespalib::VersionTagSystemRev); + printf("version tag builder: %s\n", vespalib::VersionTagBuilder); printf("nice version:\n\t"); - mbus::Vtag::printVersionNice(); + vespalib::Vtag::printVersionNice(); printf("\n"); - printf("currentVersion object: %s\n", mbus::Vtag::currentVersion.toString().c_str()); + printf("currentVersion object: %s\n", vespalib::Vtag::currentVersion.toString().c_str()); return 0; } diff --git a/messagebus/src/tests/routing/routing.cpp b/messagebus/src/tests/routing/routing.cpp index b9c673263b5..23698279137 100644 --- a/messagebus/src/tests/routing/routing.cpp +++ b/messagebus/src/tests/routing/routing.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("routing_test"); - #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/messagebus.h> @@ -16,8 +13,11 @@ LOG_SETUP("routing_test"); #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/messagebus/vtag.h> +#include <vespa/vespalib/component/vtag.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> + +LOG_SETUP("routing_test"); using namespace mbus; @@ -650,7 +650,7 @@ Test::testTrace(TestData &data, const std::vector<string> &expected) bool Test::testTrace(const std::vector<string> &expected, const Trace &trace) { - string version = Vtag::currentVersion.toString(); + string version = vespalib::Vtag::currentVersion.toString(); string actual = trace.toString(); size_t pos = 0; for (uint32_t i = 0; i < expected.size(); ++i) { diff --git a/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp b/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp index eaf609a2be1..c4268e5108d 100644 --- a/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp +++ b/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("simpleprotocol_test"); - #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/component/vtag.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/ireplyhandler.h> #include <vespa/messagebus/network/identity.h> @@ -14,7 +12,9 @@ LOG_SETUP("simpleprotocol_test"); #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/messagebus/vtag.h> +#include <vespa/log/log.h> + +LOG_SETUP("simpleprotocol_test"); using namespace mbus; @@ -25,7 +25,7 @@ Test::Main() { TEST_INIT("simpleprotocol_test"); - vespalib::Version version = Vtag::currentVersion; + vespalib::Version version = vespalib::Vtag::currentVersion; SimpleProtocol protocol; EXPECT_TRUE(protocol.getName() == "Simple"); diff --git a/messagebus/src/tests/targetpool/targetpool.cpp b/messagebus/src/tests/targetpool/targetpool.cpp index 0e63be19547..855784ec557 100644 --- a/messagebus/src/tests/targetpool/targetpool.cpp +++ b/messagebus/src/tests/targetpool/targetpool.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("targetpool_test"); - -#include <vespa/messagebus/vtag.h> #include <vespa/messagebus/network/rpctargetpool.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> + +LOG_SETUP("targetpool_test"); using namespace mbus; diff --git a/messagebus/src/vespa/messagebus/CMakeLists.txt b/messagebus/src/vespa/messagebus/CMakeLists.txt index 81b86d8e0dc..01526e0dd59 100644 --- a/messagebus/src/vespa/messagebus/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/CMakeLists.txt @@ -31,7 +31,6 @@ vespa_add_library(messagebus sourcesessionparams.cpp staticthrottlepolicy.cpp systemtimer.cpp - vtag.cpp $<TARGET_OBJECTS:messagebus_routing> $<TARGET_OBJECTS:messagebus_network> INSTALL lib64 diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 434cdb0e3c3..8ec6794919b 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -1,19 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".rpcnetwork"); - #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/iprotocol.h> #include <vespa/messagebus/tracelevel.h> -#include <vespa/messagebus/vtag.h> #include <vespa/vespalib/util/vstringfmt.h> #include <vespa/vespalib/util/sync.h> #include "inetworkowner.h" #include "rpcnetwork.h" #include "rpcsendv1.h" #include "rpcservice.h" +#include <vespa/log/log.h> + +LOG_SETUP(".rpcnetwork"); + namespace { @@ -160,7 +160,7 @@ RPCNetwork::flushTargetPool() const vespalib::Version & RPCNetwork::getVersion() const { - return Vtag::currentVersion; + return vespalib::Vtag::currentVersion; } void diff --git a/messagebus/src/vespa/messagebus/testlib/testserver.cpp b/messagebus/src/vespa/messagebus/testlib/testserver.cpp index c0ef5f9fbf9..cdc7325c508 100644 --- a/messagebus/src/vespa/messagebus/testlib/testserver.cpp +++ b/messagebus/src/vespa/messagebus/testlib/testserver.cpp @@ -1,22 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".testserver"); - +#include <vespa/vespalib/component/vtag.h> #include <vespa/vespalib/util/vstringfmt.h> #include <vespa/messagebus/network/rpcnetworkparams.h> -#include <vespa/messagebus/vtag.h> #include "oosstate.h" #include "simpleprotocol.h" #include "slobrok.h" #include "slobrokstate.h" #include "testserver.h" +#include <vespa/log/log.h> + +LOG_SETUP(".testserver"); namespace mbus { VersionedRPCNetwork::VersionedRPCNetwork(const RPCNetworkParams ¶ms) : RPCNetwork(params), - _version(Vtag::currentVersion) + _version(vespalib::Vtag::currentVersion) { // empty } diff --git a/messagebus/src/vespa/messagebus/vtag.h b/messagebus/src/vespa/messagebus/vtag.h deleted file mode 100644 index 505ffb42161..00000000000 --- a/messagebus/src/vespa/messagebus/vtag.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace vespalib { -class Version; -} - -namespace mbus { - -extern char VersionTag[]; -extern char VersionTagDate[]; -extern char VersionTagSystem[]; -extern char VersionTagSystemRev[]; -extern char VersionTagBuilder[]; - -class Vtag { -public: - static vespalib::Version currentVersion; - static void printVersionNice(); -}; - -} // namespace messagebus - diff --git a/node-admin/Dockerfile.template b/node-admin/Dockerfile.template index 485518cb86a..1e2defc97f1 100644 --- a/node-admin/Dockerfile.template +++ b/node-admin/Dockerfile.template @@ -17,8 +17,8 @@ RUN yum install -y tcpdump ADD include/root-bashrc /root/.bashrc # Override what's in the base image with local versions: -ADD target/node-admin-jar-with-dependencies.jar $VESPA_HOME/lib/jars/node-admin-jar-with-dependencies.jar ADD src/main/application/services.xml $VESPA_HOME/conf/node-admin-app/services.xml +ADD target/node-admin-jar-with-dependencies.jar $VESPA_HOME/lib/jars/node-admin-jar-with-dependencies.jar ADD scripts/configure-container-networking.py $VESPA_HOME/libexec/vespa/node-admin/configure-container-networking.py # For deploying sample application. @@ -26,11 +26,16 @@ ADD include/deploy-music-app.sh /usr/local/bin/deploy-music-app.sh ADD include/music-on-docker-services.xml $VESPA_HOME/share/vespa/sampleapps/search/music/services.xml # Entrypoint for running config server in a container. -ADD include/start-config-server.sh /usr/local/bin/start-config-server.sh +ADD include/start-config-server.sh $VESPA_HOME/bin/start-config-server.sh # Included in base image, but here overridden with local modifications. # TODO: Update the source instead. -ADD include/start-services.sh /usr/local/bin/start-services.sh +ADD include/start-services.sh $VESPA_HOME/bin/start-services.sh # Make config-server aware of node flavor 'docker'. ADD include/node-flavors.xml $VESPA_HOME/conf/configserver-app/node-flavors.xml + +# Make config-server also listen to 4080 +ADD include/http-server.xml $VESPA_HOME/conf/configserver-app/hosted-vespa/http-server.xml + +CMD $VESPA_HOME/bin/start-services.sh
\ No newline at end of file diff --git a/node-admin/include/http-server.xml b/node-admin/include/http-server.xml new file mode 100644 index 00000000000..f77d392de97 --- /dev/null +++ b/node-admin/include/http-server.xml @@ -0,0 +1 @@ +<server port="4080" id="configserver-real" />
\ No newline at end of file diff --git a/node-admin/include/nodectl-instance.sh b/node-admin/include/nodectl-instance.sh index a8d872b314e..b40ce44e33a 100755 --- a/node-admin/include/nodectl-instance.sh +++ b/node-admin/include/nodectl-instance.sh @@ -6,12 +6,14 @@ # TODO: Remove the above cookbook file (with the down-side that a new script # requires a new vespa release, instead of just a hosted release). -# Usage: nodectl-instance.sh [start|stop] +# Usage: nodectl-instance.sh [start|stop|suspend] # # start: Set the node "in service" by e.g. undraining container traffic. # start can be assumed to have completed successfully. # -# stop: Prepare for a short suspension, e.g. there's a pending upgrade. Set the +# stop: Stop services on the node (Note: Only does suspend now, will be changed soon, Oct 24 2016) +# +# suspend: Prepare for a short suspension, e.g. there's a pending upgrade. Set the # node "out of service" by draining container traffic, and flush index for a # quick start after the suspension. There's no need to stop. @@ -93,12 +95,16 @@ container_drain() { sleep 60 } -start() { +Start() { # Always start vip for now $echo $VESPA_HOME/bin/vespa-routing vip -u chef in } -stop() { +Stop() { + yinst stop +} + +Suspend() { # Always stop vip for now $echo $VESPA_HOME/bin/vespa-routing vip -u chef out @@ -113,7 +119,7 @@ stop() { main() { if [ $# -lt 1 ]; then - echo "Usage: $0 [-e] start|stop" >&2 + echo "Usage: $0 [-e] start|stop|suspend" >&2 exit 1 fi @@ -126,9 +132,11 @@ main() { action="$1" if [ "$action" = "start" ]; then - start + Start elif [ "$action" = "stop" ]; then - stop + Stop + elif [ "$action" = "suspend" ]; then + Suspend else echo "Unknown action: $action" >&2 exit 1 diff --git a/node-admin/include/start-config-server.sh b/node-admin/include/start-config-server.sh index a84b0454db3..19d1d44cc1c 100755 --- a/node-admin/include/start-config-server.sh +++ b/node-admin/include/start-config-server.sh @@ -62,19 +62,6 @@ findroot export LC_ALL=C -function WaitUntilHostIsReachable { - # Address may be IP or hostname. - local address="$1" - - echo -n "Will wait until $address is reachable... " - while ! ping -q -c 1 -W 3 "$address" &>/dev/null - do - echo "not done (will retry)" - sleep 1 - done - echo "Done" -} - function VerifyRequiredEnvironmentVariablesAreSet { if [ -z "$HOSTED_VESPA_REGION" ] then @@ -84,10 +71,6 @@ function VerifyRequiredEnvironmentVariablesAreSet { then Fail "Environment variable CONFIG_SERVER_HOSTNAME is not set" fi - if [ -z "$HOST_BRIDGE_IP" ] - then - Fail "Environment variable HOST_BRIDGE_IP is not set" - fi case "$HOSTED_VESPA_ENVIRONMENT" in prod|test|dev|staging|perf) : ;; @@ -116,10 +99,6 @@ function InternalMain { # Can also set jvmargs if necessary: # set cloudconfig_server.jvmargs=-Dvespa.freezedetector.disable=true -XX:NewRatio=1 -verbose:gc -XX:+PrintGCDateStamps -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 -Xms6g -Xmx6g - # The network is set up asynchronously and from outside of this - # container. Wait until it's done. - WaitUntilHostIsReachable "$HOST_BRIDGE_IP" - yinst start cloudconfig_server touch $VESPA_HOME/logs/jdisc_core/jdisc_core.log diff --git a/node-admin/scripts/app.sh b/node-admin/scripts/app.sh index 2757d637dc8..d3eb6996ab4 100755 --- a/node-admin/scripts/app.sh +++ b/node-admin/scripts/app.sh @@ -94,8 +94,6 @@ function DeployApp { local app_dir="$1" - VerifyApp "$app_dir" - CopyToSharedDir "$app_dir" # Create tenant diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java index 24182978cea..03b74532b49 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java @@ -32,6 +32,8 @@ public interface DockerOperations { void restartServicesOnNode(ContainerName containerName); + void stopServicesOnNode(ContainerName containerName); + void trySuspendNode(ContainerName containerName); Docker.ContainerStats getContainerStats(ContainerName containerName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index ce14b9f0caf..9537dfbe7ee 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -37,12 +37,13 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; * @author dybis */ public class DockerOperationsImpl implements DockerOperations { - private static final String NODE_PROGRAM = Defaults.getDefaults().vespaHome() + "bin/vespa-nodectl"; + private static final String NODE_PROGRAM = Defaults.getDefaults().underVespaHome("bin/vespa-nodectl"); private static final String[] GET_VESPA_VERSION_COMMAND = new String[]{NODE_PROGRAM, "vespa-version"}; private static final String[] RESUME_NODE_COMMAND = new String[] {NODE_PROGRAM, "resume"}; private static final String[] SUSPEND_NODE_COMMAND = new String[] {NODE_PROGRAM, "suspend"}; private static final String[] RESTART_NODE_COMMAND = new String[] {NODE_PROGRAM, "restart"}; + private static final String[] STOP_NODE_COMMAND = new String[] {NODE_PROGRAM, "stop"}; private static final Pattern VESPA_VERSION_PATTERN = Pattern.compile("^(\\S*)$", Pattern.MULTILINE); @@ -328,6 +329,11 @@ public class DockerOperationsImpl implements DockerOperations { } @Override + public void stopServicesOnNode(ContainerName containerName) { + executeCommandInContainer(containerName, STOP_NODE_COMMAND); + } + + @Override public Docker.ContainerStats getContainerStats(ContainerName containerName) { return docker.getContainerStats(containerName); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 65b05d9fd7c..c0de2b91cd4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -18,7 +18,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; /** - * @author valerijf + * @author freva */ public class StorageMaintainer { private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(StorageMaintainer.class); @@ -126,6 +126,10 @@ public class StorageMaintainer { Maintainer.archiveAppData(logger, containerName); } + public Maintainer getMaintainer() { + return maintainer; + } + private static class MetricsCache { private final Instant nextUpdateAt; private final Map<String, Number> metrics = new HashMap<>(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index be70dfeb0e6..585a4db5844 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -273,7 +273,6 @@ public class NodeAgentImpl implements NodeAgent { throws Exception { if (existingContainer.isRunning) { ContainerName containerName = existingContainer.name; - //PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); if (nodeSpec.nodeState == Node.State.active) { logger.info("Restarting services for " + containerName); // Since we are restarting the services we need to suspend the node. @@ -283,9 +282,15 @@ public class NodeAgentImpl implements NodeAgent { } } + private void stopServices(ContainerName containerName) throws Exception { + logger.info("Stopping services for " + containerName); + dockerOperations.stopServicesOnNode(containerName); + } + private Optional<String> shouldRemoveContainer(ContainerNodeSpec nodeSpec, Optional<Container> existingContainer) { - if (nodeSpec.nodeState != Node.State.active) { - return Optional.of("Node no longer active"); + final Node.State nodeState = nodeSpec.nodeState; + if (nodeState == Node.State.dirty || nodeState == Node.State.provisioned) { + return Optional.of("Node in state " + nodeState + ", container should no longer be running"); } if (!nodeSpec.wantedDockerImage.get().equals(existingContainer.get().image)) { return Optional.of("The node is supposed to run a new Docker image: " @@ -310,12 +315,10 @@ public class NodeAgentImpl implements NodeAgent { logger.info("Will remove container " + existingContainer.get() + ": " + removeReason.get()); if (existingContainer.get().isRunning) { - // If we're stopping the node only to upgrade we need to suspend the services. - if (nodeSpec.nodeState == Node.State.active) { - final ContainerName containerName = existingContainer.get().name; - orchestratorSuspendNode(orchestrator, nodeSpec, logger); - dockerOperations.trySuspendNode(containerName); - } + final ContainerName containerName = existingContainer.get().name; + orchestratorSuspendNode(orchestrator, nodeSpec, logger); + dockerOperations.trySuspendNode(containerName); + stopServices(containerName); } dockerOperations.removeContainer(nodeSpec, existingContainer.get(), orchestrator); return true; @@ -471,12 +474,14 @@ public class NodeAgentImpl implements NodeAgent { .add("host", hostname) .add("role", "tenants") .add("flavor", nodeSpec.nodeFlavor) - .add("state", nodeSpec.nodeState.toString()); + .add("state", nodeSpec.nodeState.toString()) + .add("zone", environment.getZone()) + .add("parentHostname", environment.getParentHostHostname()); if (nodeSpec.owner.isPresent()) { dimensionsBuilder .add("tenantName", nodeSpec.owner.get().tenant) - .add("app", nodeSpec.owner.get().application); + .add("app", nodeSpec.owner.get().application + "." + nodeSpec.owner.get().instance); } if (nodeSpec.membership.isPresent()) { dimensionsBuilder @@ -542,14 +547,16 @@ public class NodeAgentImpl implements NodeAgent { Path vespaCheckPath = Paths.get(getDefaults().underVespaHome("libexec/yms/yms_check_vespa")); SecretAgentScheduleMaker scheduleMaker = new SecretAgentScheduleMaker("vespa", 60, vespaCheckPath, "all") + .withTag("namespace", "Vespa") .withTag("role", "tenants") .withTag("flavor", nodeSpec.nodeFlavor) .withTag("state", nodeSpec.nodeState.toString()) - .withTag("zone", environment.getZone()); + .withTag("zone", environment.getZone()) + .withTag("parentHostname", environment.getParentHostHostname()); if (nodeSpec.owner.isPresent()) scheduleMaker .withTag("tenantName", nodeSpec.owner.get().tenant) - .withTag("app", nodeSpec.owner.get().application); + .withTag("app", nodeSpec.owner.get().application + "." + nodeSpec.owner.get().instance); if (nodeSpec.membership.isPresent()) scheduleMaker .withTag("clustertype", nodeSpec.membership.get().clusterType) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java index 5e33a576463..b13575d3908 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.provider; +import com.google.inject.Inject; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; @@ -44,30 +45,32 @@ public class ComponentsProviderImpl implements ComponentsProvider { // which happens rarely. Changes of apps running etc it detected by the NodeAgent. private static final int NODE_ADMIN_STATE_INTERVAL_MILLIS = 5 * 60000; - public ComponentsProviderImpl(final Docker docker, final MetricReceiverWrapper metricReceiver) { + public ComponentsProviderImpl(final Docker docker, final MetricReceiverWrapper metricReceiver, + final StorageMaintainer storageMaintainer, final Environment environment) { String baseHostName = java.util.Optional.ofNullable(System.getenv(ENV_HOSTNAME)) .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset")); - Environment environment = new Environment(); Set<String> configServerHosts = environment.getConfigServerHosts(); Orchestrator orchestrator = new OrchestratorImpl(configServerHosts); NodeRepository nodeRepository = new NodeRepositoryImpl(configServerHosts, WEB_SERVICE_PORT, baseHostName); - final Maintainer maintainer = new Maintainer(); - StorageMaintainer storageMaintainer = new StorageMaintainer(maintainer); final Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepository, - orchestrator, new DockerOperationsImpl(docker, environment, maintainer), - storageMaintainer, metricReceiver, - environment, maintainer); + orchestrator, new DockerOperationsImpl(docker, environment, storageMaintainer.getMaintainer()), + storageMaintainer, metricReceiver, environment, storageMaintainer.getMaintainer()); final NodeAdmin nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory, storageMaintainer, NODE_AGENT_SCAN_INTERVAL_MILLIS, metricReceiver); nodeAdminStateUpdater = new NodeAdminStateUpdater( nodeRepository, nodeAdmin, INITIAL_SCHEDULER_DELAY_MILLIS, NODE_ADMIN_STATE_INTERVAL_MILLIS, orchestrator, baseHostName); metricReceiverWrapper = metricReceiver; - initializeNodeAgentSecretAgent(docker, environment.getZone()); + } + + @Inject + public ComponentsProviderImpl(final Docker docker, final MetricReceiverWrapper metricReceiver) { + this(docker, metricReceiver, new StorageMaintainer(new Maintainer()), new Environment()); + initializeNodeAgentSecretAgent(docker); } @Override @@ -81,14 +84,14 @@ public class ComponentsProviderImpl implements ComponentsProvider { } - private void initializeNodeAgentSecretAgent(Docker docker, String zone) { + private void initializeNodeAgentSecretAgent(Docker docker) { ContainerName nodeAdminName = new ContainerName("node-admin"); final Path yamasAgentFolder = Paths.get("/etc/yamas-agent/"); docker.executeInContainer(nodeAdminName, "sudo", "chmod", "a+w", yamasAgentFolder.toString()); Path nodeAdminCheckPath = Paths.get("/usr/bin/curl"); SecretAgentScheduleMaker scheduleMaker = new SecretAgentScheduleMaker("node-admin", 60, nodeAdminCheckPath, - "localhost:4080/rest/metrics").withTag("zone", zone); + "localhost:4080/rest/metrics"); try { scheduleMaker.writeTo(yamasAgentFolder); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java index 94c60de3316..6af2816f653 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java @@ -9,6 +9,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.methods.HttpUriRequest; @@ -136,14 +137,21 @@ public class ConfigServerHttpRequestExecutor { } public <T> T delete(String path, int port, Class<T> wantedReturnType) { - return tryAllConfigServers(configServer -> { - return new HttpDelete("http://" + configServer + ":" + port + path); - }, wantedReturnType); + return tryAllConfigServers(configServer -> + new HttpDelete("http://" + configServer + ":" + port + path), wantedReturnType); } public <T> T get(String path, int port, Class<T> wantedReturnType) { + return tryAllConfigServers(configServer -> + new HttpGet("http://" + configServer + ":" + port + path), wantedReturnType); + } + + public <T> T post(String path, int port, Object bodyJsonPojo, Class<T> wantedReturnType) { return tryAllConfigServers(configServer -> { - return new HttpGet("http://" + configServer + ":" + port + path); + HttpPost post = new HttpPost("http://" + configServer + ":" + port + path); + setContentTypeToApplicationJson(post); + post.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo))); + return post; }, wantedReturnType); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java index 39099578843..5dda3a6b313 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.util; +import com.yahoo.net.HostName; + import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; @@ -23,22 +25,26 @@ public class Environment { private final Set<String> configServerHosts; private final String environment; private final String region; + private final String parentHostHostname; private final InetAddressResolver inetAddressResolver; public Environment() { this(getConfigServerHostsFromEnvironment(), getEnvironmentVariable(ENVIRONMENT), getEnvironmentVariable(REGION), + HostName.getLocalhost(), new InetAddressResolver()); } public Environment(Set<String> configServerHosts, String environment, String region, + String parentHostHostname, InetAddressResolver inetAddressResolver) { this.configServerHosts = configServerHosts; this.environment = environment; this.region = region; + this.parentHostHostname = parentHostHostname; this.inetAddressResolver = inetAddressResolver; } @@ -52,6 +58,10 @@ public class Environment { return region; } + public String getParentHostHostname() { + return parentHostHostname; + } + private static String getEnvironmentVariable(String name) { final String value = System.getenv(name); if (value == null) throw new IllegalStateException(String.format("Environment variable %s not set", name)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java index e105aab7b46..4cd45e3c8f2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMaker.java @@ -57,7 +57,7 @@ public class SecretAgentScheduleMaker { } } - stringBuilder.append(" tags:\n").append(" namespace: Vespa\n"); + if (!tags.isEmpty()) stringBuilder.append(" tags:\n"); tags.forEach((key, value) -> stringBuilder.append(" ").append(key).append(": ").append(value).append("\n")); return stringBuilder.toString(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 67e332a7387..c7a641feda4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -24,9 +24,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class DockerOperationsImplTest { - Environment environment = new Environment(Collections.emptySet(), + private final Environment environment = new Environment(Collections.emptySet(), "dev", "us-east-1", + "parent.host.name.yahoo.com", new InetAddressResolver()); private final Docker docker = mock(Docker.class); private final DockerOperationsImpl dockerOperations = new DockerOperationsImpl(docker, environment, new Maintainer()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java new file mode 100644 index 00000000000..fd151c5f041 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.docker; + +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.dockerapi.*; +import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; +import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.provision.Node; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * @author freva + */ +public class LocalZoneUtils { + public static final String CONFIG_SERVER_HOSTNAME = "config-server"; + public static final ContainerName CONFIG_SERVER_CONTAINER_NAME = new ContainerName(CONFIG_SERVER_HOSTNAME); + public static final int CONFIG_SERVER_WEB_SERVICE_PORT = 4080; + public static final DockerImage VESPA_LOCAL_IMAGE = new DockerImage("vespa-local:latest"); + + private static final ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create( + Collections.singleton(CONFIG_SERVER_HOSTNAME)); + private static final String APP_HOSTNAME_PREFIX = "cnode-"; + private static final String TENANT_NAME = "localtenant"; + + public static boolean startConfigServerIfNeeded(Docker docker, Environment environment) throws UnknownHostException { + Optional<Container> container = docker.getContainer(CONFIG_SERVER_HOSTNAME); + if (container.isPresent()) { + if (container.get().isRunning) return true; + else docker.deleteContainer(CONFIG_SERVER_CONTAINER_NAME); + } + + docker.createContainerCommand(VESPA_LOCAL_IMAGE, CONFIG_SERVER_CONTAINER_NAME, CONFIG_SERVER_HOSTNAME) + .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME) + .withIpAddress(environment.getInetAddressForHost(CONFIG_SERVER_HOSTNAME)) + .withVolume("/etc/hosts", "/etc/hosts") + .withEnvironment("HOSTED_VESPA_ENVIRONMENT", environment.getEnvironment()) + .withEnvironment("HOSTED_VESPA_REGION", environment.getRegion()) + .withEnvironment("CONFIG_SERVER_HOSTNAME", CONFIG_SERVER_HOSTNAME) + .withEntrypoint(Defaults.getDefaults().underVespaHome("bin/start-config-server.sh")) + .withUlimit("nofile", 16384, 16384) + .withUlimit("nproc", 409600, 409600) + .withUlimit("core", -1, -1) + .create(); + + docker.startContainer(CONFIG_SERVER_CONTAINER_NAME); + + for (int i = 0; i < 500; i++) { + try { + URL url = new URL("http://" + CONFIG_SERVER_HOSTNAME + ":" + CONFIG_SERVER_WEB_SERVICE_PORT + + "/state/v1/health"); + Thread.sleep(100); + HttpURLConnection http = (HttpURLConnection) url.openConnection(); + if (http.getResponseCode() == 200) return true; + } catch (IOException | InterruptedException ignored) { } + } + + return false; + } + + public static void buildVespaLocalDockerImage(Docker docker, DockerImage vespaBaseImage) throws IOException { + Path dockerfileTemplatePath = Paths.get("Dockerfile.template"); + Path dockerfilePath = Paths.get("Dockerfile"); + + String dockerfileTemplate = new String(Files.readAllBytes(dockerfileTemplatePath)) + .replaceAll("\\$NODE_ADMIN_FROM_IMAGE", vespaBaseImage.asString()) + .replaceAll("\\$VESPA_HOME", Defaults.getDefaults().vespaHome()); + + Files.write(dockerfilePath, dockerfileTemplate.getBytes()); + + docker.buildImage(dockerfilePath.toAbsolutePath().getParent().toFile(), VESPA_LOCAL_IMAGE); + } + + /** + * Adds numberOfNodes to node-repo and returns a set of node hostnames. + */ + public static Set<String> provisionNodes(String parentHostname, int numberOfNodes) { + Set<String> hostnames = new HashSet<>(); + List<Map<String,String>> nodesToAdd = new ArrayList<>(); + for (int i = 1; i <= numberOfNodes; i++) { + final String hostname = APP_HOSTNAME_PREFIX + i; + Map<String, String> provisionNodeRequest = new HashMap<>(); + provisionNodeRequest.put("parentHostname", parentHostname); + provisionNodeRequest.put("type", "tenant"); + provisionNodeRequest.put("flavor", "docker"); + provisionNodeRequest.put("hostname", hostname); + provisionNodeRequest.put("openStackId", "fake-" + hostname); + nodesToAdd.add(provisionNodeRequest); + hostnames.add(hostname); + } + + requestExecutor.post("/nodes/v2/node", CONFIG_SERVER_WEB_SERVICE_PORT, nodesToAdd, Map.class); + return hostnames; + } + + public static void setState(Node.State state, String hostname) { + requestExecutor.put("/nodes/v2/state/" + state + "/" + hostname, + CONFIG_SERVER_WEB_SERVICE_PORT, Optional.empty(), Map.class); + } + + public static void prepareAppForDeployment(Docker docker, Path pathToApp) { + Path pathToAppOnConfigServer = Paths.get("/tmp"); + docker.copyArchiveToContainer(pathToApp.toAbsolutePath().toString(), + CONFIG_SERVER_CONTAINER_NAME, pathToAppOnConfigServer.toString()); + + try { // Add tenant, ignore exception if tenant already exists + requestExecutor.put("/application/v2/tenant/" + TENANT_NAME, CONFIG_SERVER_WEB_SERVICE_PORT, Optional.empty(), Map.class); + } catch (RuntimeException e) { + if (! e.getMessage().contains("There already exists a tenant '" + TENANT_NAME)) { + throw e; + } + } + + final String deployPath = Defaults.getDefaults().underVespaHome("bin/deploy"); + ProcessResult copyProcess = docker.executeInContainer(CONFIG_SERVER_CONTAINER_NAME, deployPath, "-e", + TENANT_NAME, "prepare", pathToAppOnConfigServer.resolve(pathToApp.getFileName()).toString()); + if (! copyProcess.isSuccess()) { + throw new RuntimeException("Could not copy " + pathToApp + " to " + CONFIG_SERVER_CONTAINER_NAME.asString() + + "\n" + copyProcess.getErrors()); + } + + ProcessResult execProcess = docker.executeInContainer(CONFIG_SERVER_CONTAINER_NAME, deployPath, "-e", + TENANT_NAME, "activate"); + if (! execProcess.isSuccess()) { + throw new RuntimeException("Could not activate application\n" + execProcess.getErrors()); + } + } +} + diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java new file mode 100644 index 00000000000..b51a0d02f9a --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.docker; + +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.net.HostName; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.dockerapi.DockerTestUtils; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.node.admin.integrationTests.CallOrderVerifier; +import com.yahoo.vespa.hosted.node.admin.integrationTests.StorageMaintainerMock; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl; +import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; +import com.yahoo.vespa.hosted.node.maintenance.Maintainer; +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Set; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * To get started: + * 1. Add config-server and container nodes hostnames to /etc/hosts: + * $ sudo ./vespa/node-admin/scripts/etc-hosts.sh + * 2. Set environmental variables: + * VESPA_HOME="/home/y" + * VESPA_WEB_SERVICE_PORT="4080" + * 3. Create /home/docker/container-storage with read/write permissions + * 4. Update {@link RunVespaLocal#appPath} to point to the application you want deployed + * + * @author freva + */ +public class RunVespaLocal { + private static final DockerImage vespaBaseImage = + new DockerImage("docker-registry.ops.yahoo.com:4443/vespa/vespa-base:6.38.151"); + private static final Environment environment = new Environment( + Collections.singleton(LocalZoneUtils.CONFIG_SERVER_HOSTNAME), "prod", "vespa-local", + HostName.getLocalhost(), new InetAddressResolver()); + private static final Maintainer maintainer = mock(Maintainer.class); + private static Path appPath = Paths.get(System.getProperty("user.home") + "/dev/basic-search/target/application.zip"); + + + @Test + @Ignore + public void runVespaLocalTest() throws IOException, InterruptedException { + System.out.println(Defaults.getDefaults().vespaHome()); + assumeTrue(DockerTestUtils.dockerDaemonIsPresent()); + + when(maintainer.pathInHostFromPathInNode(any(), any())).thenCallRealMethod(); + when(maintainer.pathInNodeAdminToNodeCleanup(any())).thenReturn(Paths.get("/tmp")); + when(maintainer.pathInNodeAdminFromPathInNode(any(), any())).thenAnswer(invocation -> { + Object[] args = invocation.getArguments(); + return maintainer.pathInHostFromPathInNode((ContainerName) args[0], (String) args[1]); + }); + + Docker docker = DockerTestUtils.getDocker(); + LocalZoneUtils.buildVespaLocalDockerImage(docker, vespaBaseImage); + assertTrue("Could not start config server", LocalZoneUtils.startConfigServerIfNeeded(docker, environment)); + + NodeAdminStateUpdater nodeAdminStateUpdater = new ComponentsProviderImpl(docker, + new MetricReceiverWrapper(MetricReceiver.nullImplementation), + new StorageMaintainerMock(maintainer, new CallOrderVerifier()), + environment).getNodeAdminStateUpdater(); + + + try { + Set<String> hostnames = LocalZoneUtils.provisionNodes(HostName.getLocalhost(), 5); + for (String hostname : hostnames) { + try { + LocalZoneUtils.setState(Node.State.ready, hostname); + } catch (RuntimeException e) { + System.err.println(e.getMessage()); + } + } + } catch (RuntimeException e) { + System.err.println(e.getMessage()); + } + + LocalZoneUtils.prepareAppForDeployment(docker, appPath); + + while (true) { + Thread.sleep(1000); + } + +// nodeAdminStateUpdater.deconstruct(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java index 2268f45c26a..63ba0cbb7ed 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java @@ -34,6 +34,7 @@ public class ComponentsProviderWithMocks implements ComponentsProvider { private Environment environment = new Environment(Collections.emptySet(), "dev", "us-east-1", + "parent.host.name.yahoo.com", new InetAddressResolver()); private final MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); private final Function<String, NodeAgent> nodeAgentFactory = diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index 656b3e50a3a..8be3ec0fea0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; +import java.io.File; import java.net.InetAddress; import java.util.ArrayList; import java.util.Arrays; @@ -139,6 +140,11 @@ public class DockerMock implements Docker { } @Override + public void buildImage(File dockerfile, DockerImage dockerImage) { + + } + + @Override public void deleteUnusedDockerImages(Set<DockerImage> except) { } @@ -190,6 +196,11 @@ public class DockerMock implements Docker { } @Override + public CreateContainerCommand withEntrypoint(String... entrypoint) { + return this; + } + + @Override public void create() { } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 3a46ef54738..bed825bc960 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; @@ -14,10 +15,13 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; import com.yahoo.vespa.hosted.node.maintenance.Maintainer; +import com.yahoo.vespa.hosted.provision.Node; +import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collections; +import java.util.Optional; import java.util.function.Function; import static org.mockito.Matchers.any; @@ -33,14 +37,15 @@ public class DockerTester implements AutoCloseable { private final NodeRepoMock nodeRepositoryMock; private CallOrderVerifier callOrderVerifier; private Docker dockerMock; - private NodeAdminStateUpdater updater; + private final NodeAdminStateUpdater updater; private final NodeAdmin nodeAdmin; + private final OrchestratorMock orchestratorMock; public DockerTester() { callOrderVerifier = new CallOrderVerifier(); StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrderVerifier); - OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); + orchestratorMock = new OrchestratorMock(callOrderVerifier); nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); dockerMock = new DockerMock(callOrderVerifier); @@ -54,6 +59,7 @@ public class DockerTester implements AutoCloseable { Environment environment = new Environment(Collections.emptySet(), "dev", "us-east-1", + "parent.host.name.yahoo.com", inetAddressResolver); MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); @@ -71,10 +77,55 @@ public class DockerTester implements AutoCloseable { nodeRepositoryMock.addContainerNodeSpec(containerNodeSpec); } + public Optional<ContainerNodeSpec> getContainerNodeSpec(String hostname) throws IOException { + return nodeRepositoryMock.getContainerNodeSpec(hostname); + } + + public int getNumberOfContainerSpecs() { + return nodeRepositoryMock.getNumberOfContainerSpecs(); + } + + public void updateContainerNodeSpec(final String hostname, + final Optional<DockerImage> wantedDockerImage, + final ContainerName containerName, + final Node.State nodeState, + final Optional<Long> wantedRestartGeneration, + final Optional<Long> currentRestartGeneration, + final Optional<Double> minCpuCores, + final Optional<Double> minMainMemoryAvailableGb, + final Optional<Double> minDiskAvailableGb) { + + nodeRepositoryMock.updateContainerNodeSpec(hostname, + wantedDockerImage, + containerName, + nodeState, + wantedRestartGeneration, + currentRestartGeneration, + minCpuCores, + minMainMemoryAvailableGb, + minDiskAvailableGb); + } + + public void updateContainerNodeSpec(ContainerNodeSpec containerNodeSpec) { + nodeRepositoryMock.updateContainerNodeSpec(containerNodeSpec); + } + + public void clearContainerNodeSpecs() { + nodeRepositoryMock.clearContainerNodeSpecs(); + } + public NodeAdmin getNodeAdmin() { return nodeAdmin; } + public OrchestratorMock getOrchestratorMock() { + return orchestratorMock; + } + + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + return updater; + } + public CallOrderVerifier getCallOrderVerifier() { return callOrderVerifier; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index 609d876beca..d3b46966f75 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -1,132 +1,77 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.maintenance.Maintainer; import com.yahoo.vespa.hosted.provision.Node; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; import java.util.Optional; -import java.util.function.Function; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * @author valerijf */ public class MultiDockerTest { - private CallOrderVerifier callOrderVerifier; - private NodeRepoMock nodeRepositoryMock; - private DockerMock dockerMock; - private NodeAdmin nodeAdmin; - private NodeAdminStateUpdater updater; - - @Before - public void before() throws InterruptedException, UnknownHostException { - callOrderVerifier = new CallOrderVerifier(); - StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrderVerifier); - OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); - nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); - dockerMock = new DockerMock(callOrderVerifier); - - - InetAddressResolver inetAddressResolver = mock(InetAddressResolver.class); - when(inetAddressResolver.getInetAddressForHost(any(String.class))).thenReturn(InetAddress.getByName("1.1.1.1")); - Environment environment = new Environment(Collections.emptySet(), - "dev", - "us-east-1", - inetAddressResolver); - - MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - Function<String, NodeAgent> nodeAgentFactory = (hostName) -> { - final Maintainer maintainer = new Maintainer(); - return new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, - new DockerOperationsImpl(dockerMock, environment, maintainer), - maintenanceSchedulerMock, mr, environment, maintainer); - }; - nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, mr); - updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); - } - - @After - public void after() { - updater.deconstruct(); - } @Test public void test() throws InterruptedException, IOException { - addAndWaitForNode("host1", new ContainerName("container1"), Optional.of(new DockerImage("image1"))); - ContainerNodeSpec containerNodeSpec2 = - addAndWaitForNode("host2", new ContainerName("container2"), Optional.of(new DockerImage("image2"))); - - nodeRepositoryMock.updateContainerNodeSpec( - containerNodeSpec2.hostname, - containerNodeSpec2.wantedDockerImage, - containerNodeSpec2.containerName, - Node.State.dirty, - containerNodeSpec2.wantedRestartGeneration, - containerNodeSpec2.currentRestartGeneration, - containerNodeSpec2.minCpuCores, - containerNodeSpec2.minMainMemoryAvailableGb, - containerNodeSpec2.minDiskAvailableGb); - - // Wait until it is marked ready - Optional<ContainerNodeSpec> tempContainerNodeSpec; - while ((tempContainerNodeSpec = nodeRepositoryMock.getContainerNodeSpec(containerNodeSpec2.hostname)).isPresent() - && tempContainerNodeSpec.get().nodeState != Node.State.ready) { - Thread.sleep(10); + try (DockerTester dockerTester = new DockerTester()) { + addAndWaitForNode(dockerTester, "host1", new ContainerName("container1"), Optional.of(new DockerImage("image1"))); + ContainerNodeSpec containerNodeSpec2 = + addAndWaitForNode(dockerTester, "host2", new ContainerName("container2"), Optional.of(new DockerImage("image2"))); + + dockerTester.updateContainerNodeSpec( + containerNodeSpec2.hostname, + containerNodeSpec2.wantedDockerImage, + containerNodeSpec2.containerName, + Node.State.dirty, + containerNodeSpec2.wantedRestartGeneration, + containerNodeSpec2.currentRestartGeneration, + containerNodeSpec2.minCpuCores, + containerNodeSpec2.minMainMemoryAvailableGb, + containerNodeSpec2.minDiskAvailableGb); + + // Wait until it is marked ready + Optional<ContainerNodeSpec> tempContainerNodeSpec; + while ((tempContainerNodeSpec = dockerTester.getContainerNodeSpec(containerNodeSpec2.hostname)).isPresent() + && tempContainerNodeSpec.get().nodeState != Node.State.ready) { + Thread.sleep(10); + } + + addAndWaitForNode(dockerTester, "host3", new ContainerName("container3"), Optional.of(new DockerImage("image1"))); + + CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier(); + callOrderVerifier.assertInOrder( + "createContainerCommand with DockerImage: DockerImage { imageId=image1 }, HostName: host1, ContainerName: ContainerName { name=container1 }", + "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", + "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]", + + "createContainerCommand with DockerImage: DockerImage { imageId=image2 }, HostName: host2, ContainerName: ContainerName { name=container2 }", + "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", + "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]", + + "stopContainer with ContainerName: ContainerName { name=container2 }", + "deleteContainer with ContainerName: ContainerName { name=container2 }", + + "createContainerCommand with DockerImage: DockerImage { imageId=image1 }, HostName: host3, ContainerName: ContainerName { name=container3 }", + "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", + "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]"); + + callOrderVerifier.assertInOrderWithAssertMessage("Maintainer did not receive call to delete application storage", + "deleteContainer with ContainerName: ContainerName { name=container2 }", + "DeleteContainerStorage with ContainerName: ContainerName { name=container2 }"); + + callOrderVerifier.assertInOrder( + "updateNodeAttributes with HostName: host1, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=image1 }, vespaVersion='null'}", + "updateNodeAttributes with HostName: host2, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=image2 }, vespaVersion='null'}", + "markAsReady with HostName: host2", + "updateNodeAttributes with HostName: host3, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=image1 }, vespaVersion='null'}"); } - - addAndWaitForNode("host3", new ContainerName("container3"), Optional.of(new DockerImage("image1"))); - - callOrderVerifier.assertInOrder( - "createContainerCommand with DockerImage: DockerImage { imageId=image1 }, HostName: host1, ContainerName: ContainerName { name=container1 }", - "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", - "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]", - - "createContainerCommand with DockerImage: DockerImage { imageId=image2 }, HostName: host2, ContainerName: ContainerName { name=container2 }", - "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", - "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]", - - "stopContainer with ContainerName: ContainerName { name=container2 }", - "deleteContainer with ContainerName: ContainerName { name=container2 }", - - "createContainerCommand with DockerImage: DockerImage { imageId=image1 }, HostName: host3, ContainerName: ContainerName { name=container3 }", - "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", - "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]"); - - callOrderVerifier.assertInOrderWithAssertMessage("Maintainer did not receive call to delete application storage", - "deleteContainer with ContainerName: ContainerName { name=container2 }", - "DeleteContainerStorage with ContainerName: ContainerName { name=container2 }"); - - callOrderVerifier.assertInOrder( - "updateNodeAttributes with HostName: host1, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=image1 }, vespaVersion='null'}", - "updateNodeAttributes with HostName: host2, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=image2 }, vespaVersion='null'}", - "markAsReady with HostName: host2", - "updateNodeAttributes with HostName: host3, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=image1 }, vespaVersion='null'}"); } - private ContainerNodeSpec addAndWaitForNode(String hostName, ContainerName containerName, Optional<DockerImage> dockerImage) throws InterruptedException { + private ContainerNodeSpec addAndWaitForNode(DockerTester tester, String hostName, ContainerName containerName, Optional<DockerImage> dockerImage) throws InterruptedException { ContainerNodeSpec containerNodeSpec = new ContainerNodeSpec( hostName, dockerImage, @@ -142,14 +87,14 @@ public class MultiDockerTest { Optional.of(1d), Optional.of(1d), Optional.of(1d)); - nodeRepositoryMock.addContainerNodeSpec(containerNodeSpec); + tester.addContainerNodeSpec(containerNodeSpec); // Wait for node admin to be notified with node repo state and the docker container has been started - while (nodeAdmin.getListOfHosts().size() != nodeRepositoryMock.getNumberOfContainerSpecs()) { + while (tester.getNodeAdmin().getListOfHosts().size() != tester.getNumberOfContainerSpecs()) { Thread.sleep(10); } - callOrderVerifier.assertInOrder( + tester.getCallOrderVerifier().assertInOrder( "createContainerCommand with DockerImage: " + dockerImage.get() + ", HostName: " + hostName + ", ContainerName: " + containerName, "executeInContainer with ContainerName: " + containerName + ", args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", "executeInContainer with ContainerName: " + containerName + ", args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java index 280be71e253..61b2ac9fb2e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java @@ -1,177 +1,133 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.maintenance.Maintainer; import com.yahoo.vespa.hosted.provision.Node; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; import java.util.Optional; -import java.util.function.Function; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Test NodeState transitions in NodeRepository * * @author valerijf */ - public class NodeStateTest { - private CallOrderVerifier callOrderVerifier; - private NodeRepoMock nodeRepositoryMock; - private DockerMock dockerMock; - private ContainerNodeSpec initialContainerNodeSpec; - private NodeAdminStateUpdater updater; - - @Before - public void before() throws InterruptedException, UnknownHostException { - callOrderVerifier = new CallOrderVerifier(); - StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrderVerifier); - OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); - nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); - dockerMock = new DockerMock(callOrderVerifier); - - InetAddressResolver inetAddressResolver = mock(InetAddressResolver.class); - when(inetAddressResolver.getInetAddressForHost(any(String.class))).thenReturn(InetAddress.getByName("1.1.1.1")); - Environment environment = new Environment(Collections.emptySet(), - "dev", - "us-east-1", - inetAddressResolver); - - MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - Function<String, NodeAgent> nodeAgentFactory = (hostName) -> { - final Maintainer maintainer = new Maintainer(); - return new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, - new DockerOperationsImpl(dockerMock, environment, maintainer), - maintenanceSchedulerMock, mr, environment, maintainer); - }; - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, mr); - - initialContainerNodeSpec = new ContainerNodeSpec( - "host1", - Optional.of(new DockerImage("dockerImage")), - new ContainerName("container"), - Node.State.active, - "tenant", - "docker", - Optional.empty(), - Optional.empty(), - Optional.empty(), - Optional.of(1L), - Optional.of(1L), - Optional.of(1d), - Optional.of(1d), - Optional.of(1d)); - nodeRepositoryMock.addContainerNodeSpec(initialContainerNodeSpec); - - updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); + private static ContainerNodeSpec initialContainerNodeSpec = new ContainerNodeSpec( + "host1", + Optional.of(new DockerImage("dockerImage")), + new ContainerName("container"), + Node.State.active, + "tenant", + "docker", + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(1L), + Optional.of(1L), + Optional.of(1d), + Optional.of(1d), + Optional.of(1d)); + + private void setup(DockerTester tester) throws InterruptedException { + tester.addContainerNodeSpec(initialContainerNodeSpec); // Wait for node admin to be notified with node repo state and the docker container has been started - while (nodeAdmin.getListOfHosts().size() == 0) { + while (tester.getNodeAdmin().getListOfHosts().size() == 0) { Thread.sleep(10); } - callOrderVerifier.assertInOrder( + tester.getCallOrderVerifier().assertInOrder( "createContainerCommand with DockerImage: DockerImage { imageId=dockerImage }, HostName: host1, ContainerName: ContainerName { name=container }", "executeInContainer with ContainerName: ContainerName { name=container }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]"); } - @After - public void after() { - updater.deconstruct(); - } - @Test public void activeToDirty() throws InterruptedException, IOException { - // Change node state to dirty - nodeRepositoryMock.updateContainerNodeSpec( - initialContainerNodeSpec.hostname, - initialContainerNodeSpec.wantedDockerImage, - initialContainerNodeSpec.containerName, - Node.State.dirty, - initialContainerNodeSpec.wantedRestartGeneration, - initialContainerNodeSpec.currentRestartGeneration, - initialContainerNodeSpec.minCpuCores, - initialContainerNodeSpec.minMainMemoryAvailableGb, - initialContainerNodeSpec.minDiskAvailableGb); - - // Wait until it is marked ready - Optional<ContainerNodeSpec> containerNodeSpec; - while ((containerNodeSpec = nodeRepositoryMock.getContainerNodeSpec(initialContainerNodeSpec.hostname)).isPresent() - && containerNodeSpec.get().nodeState != Node.State.ready) { - Thread.sleep(10); + try (DockerTester dockerTester = new DockerTester()) { + setup(dockerTester); + // Change node state to dirty + dockerTester.updateContainerNodeSpec( + initialContainerNodeSpec.hostname, + initialContainerNodeSpec.wantedDockerImage, + initialContainerNodeSpec.containerName, + Node.State.dirty, + initialContainerNodeSpec.wantedRestartGeneration, + initialContainerNodeSpec.currentRestartGeneration, + initialContainerNodeSpec.minCpuCores, + initialContainerNodeSpec.minMainMemoryAvailableGb, + initialContainerNodeSpec.minDiskAvailableGb); + + // Wait until it is marked ready + Optional<ContainerNodeSpec> containerNodeSpec; + while ((containerNodeSpec = dockerTester.getContainerNodeSpec(initialContainerNodeSpec.hostname)).isPresent() + && containerNodeSpec.get().nodeState != Node.State.ready) { + Thread.sleep(10); + } + + assertThat(dockerTester.getContainerNodeSpec(initialContainerNodeSpec.hostname) + .get().nodeState, is(Node.State.ready)); + + dockerTester.getCallOrderVerifier() + .assertInOrder("executeInContainer with ContainerName: ContainerName { name=container }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, stop]", + "stopContainer with ContainerName: ContainerName { name=container }", + "deleteContainer with ContainerName: ContainerName { name=container }"); } - - assertThat(nodeRepositoryMock.getContainerNodeSpec(initialContainerNodeSpec.hostname).get().nodeState, is(Node.State.ready)); - - callOrderVerifier.assertInOrderWithAssertMessage("Node set to dirty, but no stop/delete call received", - "stopContainer with ContainerName: ContainerName { name=container }", - "deleteContainer with ContainerName: ContainerName { name=container }"); } @Test public void activeToInactiveToActive() throws InterruptedException, IOException { - Optional<DockerImage> newDockerImage = Optional.of(new DockerImage("newDockerImage")); - - // Change node state to inactive and change the wanted docker image - nodeRepositoryMock.updateContainerNodeSpec( - initialContainerNodeSpec.hostname, - newDockerImage, - initialContainerNodeSpec.containerName, - Node.State.inactive, - initialContainerNodeSpec.wantedRestartGeneration, - initialContainerNodeSpec.currentRestartGeneration, - initialContainerNodeSpec.minCpuCores, - initialContainerNodeSpec.minMainMemoryAvailableGb, - initialContainerNodeSpec.minDiskAvailableGb); - - callOrderVerifier.assertInOrderWithAssertMessage("Node set to inactive, but no stop/delete call received", - "stopContainer with ContainerName: ContainerName { name=container }", - "deleteContainer with ContainerName: ContainerName { name=container }"); - - - // Change node state to active - nodeRepositoryMock.updateContainerNodeSpec( - initialContainerNodeSpec.hostname, - newDockerImage, - initialContainerNodeSpec.containerName, - Node.State.active, - initialContainerNodeSpec.wantedRestartGeneration, - initialContainerNodeSpec.currentRestartGeneration, - initialContainerNodeSpec.minCpuCores, - initialContainerNodeSpec.minMainMemoryAvailableGb, - initialContainerNodeSpec.minDiskAvailableGb); - - // Check that the container is started again after the delete call - callOrderVerifier.assertInOrderWithAssertMessage("Node not started again after being put to active state", - "deleteContainer with ContainerName: ContainerName { name=container }", - "createContainerCommand with DockerImage: DockerImage { imageId=newDockerImage }, HostName: host1, ContainerName: ContainerName { name=container }", - "executeInContainer with ContainerName: ContainerName { name=container }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", - "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]"); + + try (DockerTester dockerTester = new DockerTester()) { + setup(dockerTester); + + Optional<DockerImage> newDockerImage = Optional.of(new DockerImage("newDockerImage")); + + // Change node state to inactive and change the wanted docker image + dockerTester.updateContainerNodeSpec( + initialContainerNodeSpec.hostname, + newDockerImage, + initialContainerNodeSpec.containerName, + Node.State.inactive, + initialContainerNodeSpec.wantedRestartGeneration, + initialContainerNodeSpec.currentRestartGeneration, + initialContainerNodeSpec.minCpuCores, + initialContainerNodeSpec.minMainMemoryAvailableGb, + initialContainerNodeSpec.minDiskAvailableGb); + + CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier(); + callOrderVerifier.assertInOrderWithAssertMessage("Node set to inactive, but no stop/delete call received", + "stopContainer with ContainerName: ContainerName { name=container }", + "deleteContainer with ContainerName: ContainerName { name=container }"); + + + // Change node state to active + dockerTester.updateContainerNodeSpec( + initialContainerNodeSpec.hostname, + newDockerImage, + initialContainerNodeSpec.containerName, + Node.State.active, + initialContainerNodeSpec.wantedRestartGeneration, + initialContainerNodeSpec.currentRestartGeneration, + initialContainerNodeSpec.minCpuCores, + initialContainerNodeSpec.minMainMemoryAvailableGb, + initialContainerNodeSpec.minDiskAvailableGb); + + // Check that the container is started again after the delete call + callOrderVerifier.assertInOrderWithAssertMessage("Node not started again after being put to active state", + "deleteContainer with ContainerName: ContainerName { name=container }", + "createContainerCommand with DockerImage: DockerImage { imageId=newDockerImage }, HostName: host1, ContainerName: ContainerName { name=container }", + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/usr/bin/env, test, -x, /opt/yahoo/vespa/bin/vespa-nodectl]", + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, resume]"); + } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index cef15d0ab2b..efc3938aaa5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -1,32 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.maintenance.Maintainer; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; -import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Collections; import java.util.Optional; -import java.util.function.Function; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Tests that different wanted and current restart generation leads to execution of restart command @@ -37,50 +19,29 @@ public class RestartTest { @Test public void test() throws InterruptedException, UnknownHostException { - CallOrderVerifier callOrderVerifier = new CallOrderVerifier(); - NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); - StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrderVerifier); - OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); - DockerMock dockerMock = new DockerMock(callOrderVerifier); + try (DockerTester dockerTester = new DockerTester()) { - InetAddressResolver inetAddressResolver = mock(InetAddressResolver.class); - when(inetAddressResolver.getInetAddressForHost(any(String.class))).thenReturn(InetAddress.getByName("1.1.1.1")); - Environment environment = new Environment(Collections.emptySet(), - "dev", - "us-east-1", - inetAddressResolver); + long wantedRestartGeneration = 1; + long currentRestartGeneration = wantedRestartGeneration; + dockerTester.addContainerNodeSpec(createContainerNodeSpec(wantedRestartGeneration, currentRestartGeneration)); - MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - Function<String, NodeAgent> nodeAgentFactory = (hostName) -> { - final Maintainer maintainer = new Maintainer(); - return new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, - new DockerOperationsImpl(dockerMock, environment, maintainer), - maintenanceSchedulerMock, mr, environment, maintainer); - }; - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, mr); + // Wait for node admin to be notified with node repo state and the docker container has been started + while (dockerTester.getNodeAdmin().getListOfHosts().size() == 0) { + Thread.sleep(10); + } - long wantedRestartGeneration = 1; - long currentRestartGeneration = wantedRestartGeneration; - nodeRepositoryMock.addContainerNodeSpec(createContainerNodeSpec(wantedRestartGeneration, currentRestartGeneration)); + CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier(); + // Check that the container is started and NodeRepo has received the PATCH update + callOrderVerifier.assertInOrder("createContainerCommand with DockerImage: DockerImage { imageId=dockerImage }, HostName: host1, ContainerName: ContainerName { name=container }", + "updateNodeAttributes with HostName: host1, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=dockerImage }, vespaVersion='null'}"); - NodeAdminStateUpdater updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); + wantedRestartGeneration = 2; + currentRestartGeneration = 1; + dockerTester.updateContainerNodeSpec(createContainerNodeSpec(wantedRestartGeneration, currentRestartGeneration)); - // Wait for node admin to be notified with node repo state and the docker container has been started - while (nodeAdmin.getListOfHosts().size() == 0) { - Thread.sleep(10); + callOrderVerifier.assertInOrder("Suspend for host1", + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, restart]"); } - - // Check that the container is started and NodeRepo has received the PATCH update - callOrderVerifier.assertInOrder("createContainerCommand with DockerImage: DockerImage { imageId=dockerImage }, HostName: host1, ContainerName: ContainerName { name=container }", - "updateNodeAttributes with HostName: host1, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=dockerImage }, vespaVersion='null'}"); - - wantedRestartGeneration = 2; - currentRestartGeneration = 1; - nodeRepositoryMock.updateContainerNodeSpec(createContainerNodeSpec(wantedRestartGeneration, currentRestartGeneration)); - - callOrderVerifier.assertInOrder("Suspend for host1", - "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/yahoo/vespa/bin/vespa-nodectl, restart]"); - updater.deconstruct(); } private ContainerNodeSpec createContainerNodeSpec(long wantedRestartGeneration, long currentRestartGeneration) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java index 1f3bc1244ab..d899b655a39 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java @@ -1,34 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.maintenance.Maintainer; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; -import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Collections; import java.util.Optional; -import java.util.function.Function; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Scenario test for NodeAdminStateUpdater. @@ -38,88 +23,72 @@ import static org.mockito.Mockito.when; public class ResumeTest { @Test public void test() throws InterruptedException, UnknownHostException { - CallOrderVerifier callOrderVerifier = new CallOrderVerifier(); - NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); - StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrderVerifier); - OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); - DockerMock dockerMock = new DockerMock(callOrderVerifier); - - InetAddressResolver inetAddressResolver = mock(InetAddressResolver.class); - when(inetAddressResolver.getInetAddressForHost(any(String.class))).thenReturn(InetAddress.getByName("1.1.1.1")); - Environment environment = new Environment(Collections.emptySet(), - "dev", - "us-east-1", - inetAddressResolver); - - MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - Function<String, NodeAgent> nodeAgentFactory = (hostName) -> { - final Maintainer maintainer = new Maintainer(); - return new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, - new DockerOperationsImpl(dockerMock, environment, maintainer), - maintenanceSchedulerMock, mr, environment, maintainer); - }; - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, mr); - - nodeRepositoryMock.addContainerNodeSpec(new ContainerNodeSpec( - "host1", - Optional.of(new DockerImage("dockerImage")), - new ContainerName("container"), - Node.State.active, - "tenant", - "docker", - Optional.empty(), - Optional.empty(), - Optional.empty(), - Optional.of(1L), - Optional.of(1L), - Optional.of(1d), - Optional.of(1d), - Optional.of(1d))); - - NodeAdminStateUpdater updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); - - // Wait for node admin to be notified with node repo state and the docker container has been started - while (nodeAdmin.getListOfHosts().size() == 0) { + try (DockerTester dockerTester = new DockerTester()) { + final NodeAdmin nodeAdmin = dockerTester.getNodeAdmin(); + final OrchestratorMock orchestratorMock = dockerTester.getOrchestratorMock(); + final NodeAdminStateUpdater nodeAdminStateUpdater = dockerTester.getNodeAdminStateUpdater(); + + dockerTester.addContainerNodeSpec(new ContainerNodeSpec( + "host1", + Optional.of(new DockerImage("dockerImage")), + new ContainerName("container"), + Node.State.active, + "tenant", + "docker", + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(1L), + Optional.of(1L), + Optional.of(1d), + Optional.of(1d), + Optional.of(1d))); + + // Wait for node admin to be notified with node repo state and the docker container has been started + + while (nodeAdmin.getListOfHosts().size() == 0) { + Thread.sleep(10); + } + + CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier(); + // Check that the container is started and NodeRepo has received the PATCH update + callOrderVerifier.assertInOrder("createContainerCommand with DockerImage: DockerImage { imageId=dockerImage }, HostName: host1, ContainerName: ContainerName { name=container }", + "updateNodeAttributes with HostName: host1, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=dockerImage }, vespaVersion='null'}"); + + // Force orchestrator to reject the suspend + orchestratorMock.setForceGroupSuspendResponse(Optional.of("Orchestrator reject suspend")); + + // At this point NodeAdmin should be fine with the suspend and it is up to Orchestrator + while (!nodeAdminStateUpdater + .setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED) + .equals(Optional.of("Orchestrator reject suspend"))) { + Thread.sleep(10); + } + assertThat(nodeAdminStateUpdater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.of("Orchestrator reject suspend"))); + + //Make orchestrator allow suspend requests + orchestratorMock.setForceGroupSuspendResponse(Optional.empty()); + assertThat(nodeAdminStateUpdater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.empty())); + + // Now, change data in node repo, should not propagate. + dockerTester.clearContainerNodeSpecs(); + + // New node repo state should have not propagated to node admin Thread.sleep(10); - } - - // Check that the container is started and NodeRepo has received the PATCH update - callOrderVerifier.assertInOrder("createContainerCommand with DockerImage: DockerImage { imageId=dockerImage }, HostName: host1, ContainerName: ContainerName { name=container }", - "updateNodeAttributes with HostName: host1, NodeAttributes: NodeAttributes{restartGeneration=1, dockerImage=DockerImage { imageId=dockerImage }, vespaVersion='null'}"); - - // Force orchestrator to reject the suspend - orchestratorMock.setForceGroupSuspendResponse(Optional.of("Orchestrator reject suspend")); - - // At this point NodeAdmin should be fine with the suspend and it is up to Orchestrator - while (!updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED) - .equals(Optional.of("Orchestrator reject suspend"))) { - Thread.sleep(10); - } - assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.of("Orchestrator reject suspend"))); - - //Make orchestrator allow suspend requests - orchestratorMock.setForceGroupSuspendResponse(Optional.empty()); - assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.empty())); + assertThat(nodeAdmin.getListOfHosts().size(), is(1)); - // Now, change data in node repo, should not propagate. - nodeRepositoryMock.clearContainerNodeSpecs(); + // Now resume + assertThat(nodeAdminStateUpdater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED), is(Optional.empty())); - // New node repo state should have not propagated to node admin - Thread.sleep(10); - assertThat(nodeAdmin.getListOfHosts().size(), is(1)); + // Now node repo state should propagate to node admin again + while (nodeAdmin.getListOfHosts().size() != 0) { + Thread.sleep(10); + } - // Now resume - assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED), is(Optional.empty())); + callOrderVerifier.assertInOrder("Resume for host1", + "Suspend with parent: basehostname and hostnames: [host1] - Forced response: Optional[Orchestrator reject suspend]", + "Suspend with parent: basehostname and hostnames: [host1] - Forced response: Optional.empty"); - // Now node repo state should propagate to node admin again - while (nodeAdmin.getListOfHosts().size() != 0) { - Thread.sleep(10); } - - callOrderVerifier.assertInOrder("Resume for host1", - "Suspend with parent: basehostname and hostnames: [host1] - Forced response: Optional[Orchestrator reject suspend]", - "Suspend with parent: basehostname and hostnames: [host1] - Forced response: Optional.empty"); - - updater.deconstruct(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index 3e5821001c9..61eb4a87c32 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -16,7 +16,11 @@ public class StorageMaintainerMock extends StorageMaintainer { private final CallOrderVerifier callOrderVerifier; public StorageMaintainerMock(CallOrderVerifier callOrderVerifier) { - super(new Maintainer()); + this(new Maintainer(), callOrderVerifier); + } + + public StorageMaintainerMock(Maintainer maintainer, CallOrderVerifier callOrderVerifier) { + super(maintainer); this.callOrderVerifier = callOrderVerifier; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 67af5a996af..600954fa1d8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -64,6 +64,7 @@ public class NodeAgentImplTest { Environment environment = new Environment(Collections.emptySet(), "dev", "us-east-1", + "parent.host.name.yahoo.com", new InetAddressResolver()); private final NodeAgentImpl nodeAgent = new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, storageMaintainer, metricReceiver, environment, maintainer); @@ -235,7 +236,7 @@ public class NodeAgentImplTest { } @Test - public void failedNodeRunningContainerIsTakenDown() throws Exception { + public void failedNodeRunningContainerShouldStillBeRunning() throws Exception { final long restartGeneration = 1; final DockerImage dockerImage = new DockerImage("dockerImage"); final ContainerName containerName = new ContainerName("container-name"); @@ -260,14 +261,14 @@ public class NodeAgentImplTest { nodeAgent.tick(); - verify(dockerOperations, times(1)).removeContainer(any(), any(), any()); - verify(dockerOperations, times(1)).removeContainer(eq(nodeSpec), any(), any()); + verify(dockerOperations, never()).removeContainer(any(), any(), any()); + verify(dockerOperations, never()).removeContainer(eq(nodeSpec), any(), any()); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); } @Test - public void inactiveNodeRunningContainerIsTakenDown() throws Exception { + public void inactiveNodeRunningContainerShouldStillBeRunning() throws Exception { final long restartGeneration = 1; final DockerImage dockerImage = new DockerImage("dockerImage"); final ContainerName containerName = new ContainerName("container-name"); @@ -294,7 +295,7 @@ public class NodeAgentImplTest { final InOrder inOrder = inOrder(storageMaintainer, dockerOperations); inOrder.verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(containerName)); - inOrder.verify(dockerOperations, times(1)).removeContainer(eq(nodeSpec), any(), any()); + inOrder.verify(dockerOperations, never()).removeContainer(eq(nodeSpec), any(), any()); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); @@ -322,7 +323,10 @@ public class NodeAgentImplTest { when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); when(dockerOperations.getContainer(eq(hostName))).thenReturn( - Optional.of(new Container(hostName, dockerImage, containerName, nodeState != Node.State.dirty))); + Optional.of(new Container(hostName, + dockerImage, + containerName, + nodeState != Node.State.dirty && nodeState != Node.State.provisioned))); nodeAgent.tick(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMakerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMakerTest.java index 471101a85dc..3699b48a67b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMakerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentScheduleMakerTest.java @@ -35,7 +35,6 @@ public class SecretAgentScheduleMakerTest { " - arg1\n" + " - arg2 with space\n" + " tags:\n" + - " namespace: Vespa\n" + " tenantName: vespa\n" + " app: canary-docker.default\n" + " clustertype: container\n" + @@ -56,9 +55,7 @@ public class SecretAgentScheduleMakerTest { "- id: system-checks\n" + " interval: 60\n" + " user: nobody\n" + - " check: /some/test\n" + - " tags:\n" + - " namespace: Vespa\n", scheduleMaker.toString()); + " check: /some/test\n", scheduleMaker.toString()); } @Test @@ -70,8 +67,6 @@ public class SecretAgentScheduleMakerTest { "- id: system-checks\n" + " interval: 60\n" + " user: yahoo\n" + - " check: /some/test\n" + - " tags:\n" + - " namespace: Vespa\n", scheduleMaker.toString()); + " check: /some/test\n", scheduleMaker.toString()); } } diff --git a/node-admin/src/test/resources/docker.stats.metrics.expected.json b/node-admin/src/test/resources/docker.stats.metrics.expected.json index c8bdf87707c..50a47a9e14a 100644 --- a/node-admin/src/test/resources/docker.stats.metrics.expected.json +++ b/node-admin/src/test/resources/docker.stats.metrics.expected.json @@ -3,7 +3,7 @@ "application":"docker", "dimensions":{ "flavor":"docker", - "app":"testapp", + "app":"testapp.testinstance", "clustertype":"clustType", "role":"tenants", "tenantName":"tester", @@ -11,38 +11,52 @@ "vespaVersion":"1.2.3", "state":"active", "clusterid":"clustId", + "parentHostname":"parent.host.name.yahoo.com", + "zone":"dev.us-east-1", "interface":"eth1" }, "metrics":{ "node.network.bytes_sent":5.4246745E7, "node.network.bytes_rcvd":3245766.0 + }, + "routing":{ + "yamas":{ + "namespaces": ["Vespa"] + } } }, { "application":"docker", "dimensions":{ "flavor":"docker", - "app":"testapp", + "app":"testapp.testinstance", "clustertype":"clustType", "role":"tenants", "tenantName":"tester", "host":"hostname", "vespaVersion":"1.2.3", "state":"active", - "clusterid":"clustId" + "clusterid":"clustId", + "parentHostname":"parent.host.name.yahoo.com", + "zone":"dev.us-east-1" }, "metrics":{ "node.cpu.busy.pct": 45.6789123, "node.cpu.throttled_time": 4523.0, "node.memory.usage":1.752707072E9, "node.memory.limit":4.294967296E9 + }, + "routing":{ + "yamas":{ + "namespaces": ["Vespa"] + } } }, { "application":"docker", "dimensions":{ "flavor":"docker", - "app":"testapp", + "app":"testapp.testinstance", "clustertype":"clustType", "role":"tenants", "tenantName":"tester", @@ -50,11 +64,18 @@ "vespaVersion":"1.2.3", "state":"active", "clusterid":"clustId", + "parentHostname":"parent.host.name.yahoo.com", + "zone":"dev.us-east-1", "interface":"eth0" }, "metrics":{ "node.network.bytes_sent":2.0303455E7, "node.network.bytes_rcvd":1.949927E7 + }, + "routing":{ + "yamas":{ + "namespaces": ["Vespa"] + } } } -]
\ No newline at end of file +] diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index d85347847ae..f2d50e76baa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -12,7 +12,6 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.slime.Type; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -45,7 +44,6 @@ public class NodeSerializer { private static final String hostnameKey = "hostname"; private static final String openStackIdKey = "openStackId"; private static final String parentHostnameKey = "parentHostname"; - private static final String configurationKey ="configuration"; // TODO: Remove when 6.31 is deployed everywhere private static final String historyKey = "history"; private static final String instanceKey = "instance"; // legacy name, TODO: change to allocation with backwards compat private static final String rebootGenerationKey = "rebootGeneration"; @@ -68,7 +66,7 @@ public class NodeSerializer { private static final String restartGenerationKey = "restartGeneration"; private static final String currentRestartGenerationKey = "currentRestartGeneration"; private static final String removableKey = "removable"; - //Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable. + // Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable. private static final String dockerImageKey = "dockerImage"; // History event fields @@ -164,7 +162,6 @@ public class NodeSerializer { } private Flavor flavorFromSlime(Inspector object) { - if (object.field(configurationKey).valid()) object = object.field(configurationKey); // TODO: Remove this line when 6.31 is deployed everywhere return flavors.getFlavorOrThrow(object.field(flavorKey).asString()); } @@ -223,13 +220,6 @@ public class NodeSerializer { private Optional<Status.HardwareFailureType> hardwareFailureFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); - // TODO: Remove boolean handling when 6.28 is deployed everywhere - if (object.type() == Type.BOOL) { - if (!object.asBool()) { - return Optional.empty(); - } - return Optional.of(Status.HardwareFailureType.unknown); - } return Optional.of(hardwareFailureFromString(object.asString())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index b637c4ba1bc..ce584c0bce7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -216,7 +216,7 @@ class GroupPreparer { if (offered.allocation().isPresent()) { ClusterMembership membership = offered.allocation().get().membership(); if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application - if ( ! membership.cluster().equalsIgnoringGroup(cluster)) continue; // wrong cluster id/type + if ( ! membership.cluster().equalsIgnoringGroupAndDockerImage(cluster)) continue; // wrong cluster id/type if ((! canChangeGroup || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index f9f44fb55b2..0763d70f419 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -26,7 +26,7 @@ public class CuratorDatabaseClientTest { @Test public void ensure_can_read_stored_host_information() throws Exception { - String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159, \"type\":\"host\"}"; + String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"flavor\":\"default\",\"created\":1421054425159, \"type\":\"host\"}"; curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes()); List<Node> allocatedNodes = zkClient.getNodes(Node.State.ready); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index f05143a05b6..56d19446cd9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -144,9 +144,7 @@ public class SerializationTest { "{\n" + " \"type\" : \"tenant\",\n" + " \"rebootGeneration\" : 0,\n" + - " \"configuration\" : {\n" + - " \"flavor\" : \"large\"\n" + - " },\n" + + " \"flavor\" : \"large\"\n," + " \"history\" : [\n" + " {\n" + " \"type\" : \"reserved\",\n" + @@ -163,79 +161,13 @@ public class SerializationTest { " },\n" + " \"openStackId\" : \"myId\",\n" + " \"hostname\" : \"myHostname\",\n" + - " \"hardwareFailure\" : true\n" + + " \"hardwareFailure\" : \"memory_mcelog\"\n" + "}"; Node node = nodeSerializer.fromJson(Node.State.provisioned, Utf8.toBytes(nodeData)); assertEquals("large", node.flavor().canonicalName()); } - // TODO: Remove when 6.28 is deployed everywhere - @Test - public void testLegacyHardwareFailureBooleanDeserialization() { - String nodeData = - "{\n" + - " \"type\" : \"tenant\",\n" + - " \"rebootGeneration\" : 0,\n" + - " \"configuration\" : {\n" + - " \"flavor\" : \"default\"\n" + - " },\n" + - " \"history\" : [\n" + - " {\n" + - " \"type\" : \"reserved\",\n" + - " \"at\" : 1444391402611\n" + - " }\n" + - " ],\n" + - " \"instance\" : {\n" + - " \"applicationId\" : \"myApplication\",\n" + - " \"tenantId\" : \"myTenant\",\n" + - " \"instanceId\" : \"myInstance\",\n" + - " \"serviceId\" : \"content/myId/0\",\n" + - " \"restartGeneration\" : 0,\n" + - " \"removable\" : false\n" + - " },\n" + - " \"openStackId\" : \"myId\",\n" + - " \"hostname\" : \"myHostname\",\n" + - " \"hardwareFailure\" : true\n" + - "}"; - - Node node = nodeSerializer.fromJson(Node.State.provisioned, Utf8.toBytes(nodeData)); - assertEquals(Status.HardwareFailureType.unknown, node.status().hardwareFailure().get()); - } - - // TODO: Remove when 6.28 is deployed everywhere - @Test - public void testLegacyNonHardwareFailureDeserialization() { - String nodeData = - "{\n" + - " \"type\" : \"tenant\",\n" + - " \"rebootGeneration\" : 0,\n" + - " \"configuration\" : {\n" + - " \"flavor\" : \"default\"\n" + - " },\n" + - " \"history\" : [\n" + - " {\n" + - " \"type\" : \"reserved\",\n" + - " \"at\" : 1444391402611\n" + - " }\n" + - " ],\n" + - " \"instance\" : {\n" + - " \"applicationId\" : \"myApplication\",\n" + - " \"tenantId\" : \"myTenant\",\n" + - " \"instanceId\" : \"myInstance\",\n" + - " \"serviceId\" : \"content/myId/0\",\n" + - " \"restartGeneration\" : 0,\n" + - " \"removable\" : false\n" + - " },\n" + - " \"openStackId\" : \"myId\",\n" + - " \"hostname\" : \"myHostname\",\n" + - " \"hardwareFailure\" : false\n" + - "}"; - - Node node = nodeSerializer.fromJson(Node.State.provisioned, Utf8.toBytes(nodeData)); - assertFalse(node.status().hardwareFailure().isPresent()); - } - @Test public void testRetiredNodeSerialization() { Node node = createNode(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 5e43fa37fb3..fe37c3bbf60 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -34,12 +34,27 @@ public class DockerProvisioningTest { tester.makeReadyDockerNodes(1, dockerFlavor, "dockerHost" + i); } - List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Optional.empty()), 7, 1, dockerFlavor); + Optional<String> wantedDockerImage = Optional.of("docker-registry.ops.yahoo.com:4443/vespa/vespa-base:6.39"); + final int nodeCount = 7; + List<HostSpec> hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedDockerImage), + nodeCount, 1, dockerFlavor); tester.activate(application1, new HashSet<>(hosts)); final NodeList nodes = tester.getNodes(application1, Node.State.active); - assertEquals(7, nodes.size()); + assertEquals(nodeCount, nodes.size()); assertEquals(dockerFlavor, nodes.asList().get(0).flavor().canonicalName()); + + // Upgrade Vespa version on nodes + Optional<String> upgradedWantedDockerImage = Optional.of("docker-registry.ops.yahoo.com:4443/vespa/vespa-base:6.40"); + List<HostSpec> upgradedHosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), upgradedWantedDockerImage), + nodeCount, 1, dockerFlavor); + tester.activate(application1, new HashSet<>(upgradedHosts)); + final NodeList upgradedNodes = tester.getNodes(application1, Node.State.active); + assertEquals(nodeCount, upgradedNodes.size()); + assertEquals(dockerFlavor, upgradedNodes.asList().get(0).flavor().canonicalName()); + assertEquals(hosts, upgradedHosts); } // In dev, test and staging you get nodes with default flavor, but we should get specified flavor for docker nodes diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 6aea7ae4d61..e3f27bc31d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -189,7 +189,7 @@ public class ProvisioningTester implements AutoCloseable { Set<Integer> indices = new HashSet<>(); for (HostSpec host : hosts) { ClusterSpec nodeCluster = host.membership().get().cluster(); - assertTrue(requestedCluster.equalsIgnoringGroup(nodeCluster)); + assertTrue(requestedCluster.equalsIgnoringGroupAndDockerImage(nodeCluster)); if (requestedCluster.group().isPresent()) assertEquals(requestedCluster.group(), nodeCluster.group()); else diff --git a/persistence/src/vespa/persistence/proxy/buildid.cpp b/persistence/src/vespa/persistence/proxy/buildid.cpp index e102288610c..c78af647de2 100644 --- a/persistence/src/vespa/persistence/proxy/buildid.cpp +++ b/persistence/src/vespa/persistence/proxy/buildid.cpp @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> +#include <vespa/vespalib/component/vtag.h> #include "buildid.h" const char *storage::spi::getBuildId() { - return V_TAG_COMPONENT; + return vespalib::VersionTagComponent; } @@ -1119,6 +1119,7 @@ <module>config-proxy</module> <module>configserver</module> <module>config_test</module> + <module>container</module> <module>container-core</module> <module>container-accesslogging</module> <module>container-dev</module> diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 7304593bda9..9e4d0c4f92c 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -5,17 +5,21 @@ LOG_SETUP("tensorattribute_test"); #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/tensor/generic_tensor_attribute.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/vespalib/tensor/tensor_factory.h> #include <vespa/vespalib/tensor/default_tensor.h> +#include <vespa/vespalib/io/fileutil.h> using search::attribute::TensorAttribute; +using search::attribute::DenseTensorAttribute; using search::attribute::GenericTensorAttribute; using search::AttributeGuard; using search::AttributeVector; using vespalib::eval::ValueType; using vespalib::tensor::Tensor; using vespalib::tensor::TensorCells; +using vespalib::tensor::DenseTensorCells; using vespalib::tensor::TensorDimensions; using vespalib::tensor::TensorFactory; @@ -30,6 +34,11 @@ static bool operator==(const Tensor &lhs, const Tensor &rhs) } } +vespalib::string sparseSpec("tensor(x{},y{})"); +vespalib::string denseSpec("tensor(x[2],y[3])"); +vespalib::string denseAbstractSpec_xy("tensor(x[],y[])"); +vespalib::string denseAbstractSpec_x("tensor(x[2],y[])"); +vespalib::string denseAbstractSpec_y("tensor(x[],y[3])"); struct Fixture { @@ -39,22 +48,42 @@ struct Fixture Config _cfg; vespalib::string _name; + vespalib::string _typeSpec; std::shared_ptr<TensorAttribute> _tensorAttr; std::shared_ptr<AttributeVector> _attr; vespalib::tensor::DefaultTensor::builder _builder; + bool _denseTensors; + bool _useDenseTensorAttribute; - Fixture(const vespalib::string &typeSpec) + Fixture(const vespalib::string &typeSpec, + bool useDenseTensorAttribute = false) : _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name("test"), + _typeSpec(typeSpec), _tensorAttr(), - _attr() + _attr(), + _builder(), + _denseTensors(false), + _useDenseTensorAttribute(useDenseTensorAttribute) { _cfg.setTensorType(ValueType::from_spec(typeSpec)); - _tensorAttr = std::make_shared<GenericTensorAttribute>(_name, _cfg); + if (_cfg.tensorType().is_dense()) { + _denseTensors = true; + } + _tensorAttr = makeAttr(); _attr = _tensorAttr; _attr->addReservedDoc(); } + std::shared_ptr<TensorAttribute> makeAttr() { + if (_useDenseTensorAttribute) { + assert(_denseTensors); + return std::make_shared<DenseTensorAttribute>(_name, _cfg); + } else { + return std::make_shared<GenericTensorAttribute>(_name, _cfg); + } + } + Tensor::UP createTensor(const TensorCells &cells) { return TensorFactory::create(cells, _builder); } @@ -62,6 +91,9 @@ struct Fixture const TensorDimensions &dimensions) { return TensorFactory::create(cells, dimensions, _builder); } + Tensor::UP createDenseTensor(const DenseTensorCells &cells) const { + return TensorFactory::createDense(cells); + } void ensureSpace(uint32_t docId) { while (_attr->getNumDocs() <= docId) { @@ -113,78 +145,174 @@ struct Fixture assertGetTensor(*expTensor, docId); } + void + assertGetDenseTensor(const DenseTensorCells &expCells, + uint32_t docId) + { + Tensor::UP expTensor = createDenseTensor(expCells); + assertGetTensor(*expTensor, docId); + } + void save() { bool saveok = _attr->save(); EXPECT_TRUE(saveok); } void load() { - _tensorAttr = std::make_shared<GenericTensorAttribute>(_name, _cfg); + _tensorAttr = makeAttr(); _attr = _tensorAttr; bool loadok = _attr->load(); EXPECT_TRUE(loadok); } + + bool isUnbound(const vespalib::string &dimensionName) const + { + ValueType type = _cfg.tensorType(); + for (const auto &dim : type.dimensions()) { + if (dim.name == dimensionName && !dim.is_bound()) { + return true; + } + } + return false; + } + + Tensor::UP expDenseTensor3() const + { + if (isUnbound("x")) { + if (isUnbound("y")) { + return createDenseTensor({ {{{"x",0},{"y",1}}, 11} }); + } + return createDenseTensor({ {{{"x",0},{"y",1}}, 11}, + {{{"x",0},{"y",2}}, 0} }); + } else if (isUnbound("y")) { + return createDenseTensor({ {{{"x",0},{"y",1}}, 11}, + {{{"x",1},{"y",0}}, 0} }); + } + return createDenseTensor({ {{{"x",0},{"y",1}}, 11}, + {{{"x",1},{"y",2}}, 0} }); + } + + Tensor::UP expDenseFillTensor() const + { + if (isUnbound("x")) { + if (isUnbound("y")) { + return createDenseTensor({ {{{"x",0},{"y",0}}, 5} }); + } + return createDenseTensor({ {{{"x",0},{"y",0}}, 5}, + {{{"x",0},{"y",2}}, 0} }); + } else if (isUnbound("y")) { + return createDenseTensor({ {{{"x",0},{"y",0}}, 5}, + {{{"x",1},{"y",0}}, 0} }); + } + return createDenseTensor({ {{{"x",0},{"y",0}}, 5}, + {{{"x",1},{"y",2}}, 0} }); + } + + Tensor::UP expEmptyDenseTensor() const + { + if (isUnbound("x")) { + if (isUnbound("y")) { + return createDenseTensor({ {{{"x",0},{"y",0}}, 0} }); + } + return createDenseTensor({ {{{"x",0},{"y",2}}, 0} }); + } else if (isUnbound("y")) { + return createDenseTensor({ {{{"x",1},{"y",0}}, 0} }); + } + return createDenseTensor({ {{{"x",1},{"y",2}}, 0} }); + } + + vespalib::string expEmptyDenseTensorSpec() const { + if (isUnbound("x")) { + if (isUnbound("y")) { + return "tensor(x[1],y[1])"; + } + return "tensor(x[1],y[3])"; + } else if (isUnbound("y")) { + return "tensor(x[2],y[1])"; + } + return "tensor(x[2],y[3])"; + } + + void testEmptyAttribute(); + void testSetTensorValue(); + void testSaveLoad(); + void testCompaction(); + void testTensorTypeFileHeaderTag(); + void testEmptyTensor(); }; -TEST_F("Test empty tensor attribute", Fixture("tensor()")) +void +Fixture::testEmptyAttribute() { - EXPECT_EQUAL(1u, f._attr->getNumDocs()); - EXPECT_EQUAL(1u, f._attr->getCommittedDocIdLimit()); + EXPECT_EQUAL(1u, _attr->getNumDocs()); + EXPECT_EQUAL(1u, _attr->getCommittedDocIdLimit()); } - -TEST_F("Test setting tensor value", Fixture("tensor(x{}, y{})")) +void +Fixture::testSetTensorValue() { - f.ensureSpace(4); - EXPECT_EQUAL(5u, f._attr->getNumDocs()); - EXPECT_EQUAL(5u, f._attr->getCommittedDocIdLimit()); - TEST_DO(f.assertGetNoTensor(4)); - f.setTensor(4, *f.createTensor({}, {})); - TEST_DO(f.assertGetTensor({}, {"x", "y"}, 4)); - f.setTensor(3, *f.createTensor({ {{}, 3} }, { "x", "y"})); - TEST_DO(f.assertGetTensor({ {{}, 3} }, { "x", "y"}, 3)); - TEST_DO(f.assertGetNoTensor(2)); - TEST_DO(f.clearTensor(3)); - TEST_DO(f.assertGetNoTensor(3)); + ensureSpace(4); + EXPECT_EQUAL(5u, _attr->getNumDocs()); + EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); + TEST_DO(assertGetNoTensor(4)); + setTensor(4, *createTensor({}, {})); + if (_denseTensors) { + TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4)); + setTensor(3, *createTensor({ {{{"y","1"}}, 11} }, { "x", "y"})); + TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); + } else { + TEST_DO(assertGetTensor({}, {"x", "y"}, 4)); + setTensor(3, *createTensor({ {{}, 11} }, { "x", "y"})); + TEST_DO(assertGetTensor({ {{}, 11} }, { "x", "y"}, 3)); + } + TEST_DO(assertGetNoTensor(2)); + TEST_DO(clearTensor(3)); + TEST_DO(assertGetNoTensor(3)); } - -TEST_F("Test saving / loading tensor attribute", Fixture("tensor(x{}, y{})")) +void +Fixture::testSaveLoad() { - f.ensureSpace(4); - f.setTensor(4, *f.createTensor({}, {})); - f.setTensor(3, *f.createTensor({ {{}, 3} }, { "x", "y"})); - TEST_DO(f.save()); - TEST_DO(f.load()); - EXPECT_EQUAL(5u, f._attr->getNumDocs()); - EXPECT_EQUAL(5u, f._attr->getCommittedDocIdLimit()); - TEST_DO(f.assertGetTensor({ {{}, 3} }, { "x", "y"}, 3)); - TEST_DO(f.assertGetTensor({}, {"x", "y"}, 4)); - TEST_DO(f.assertGetNoTensor(2)); + ensureSpace(4); + setTensor(4, *createTensor({}, {})); + setTensor(3, *createTensor({ {{{"y","1"}}, 11} }, { "x", "y"})); + TEST_DO(save()); + TEST_DO(load()); + EXPECT_EQUAL(5u, _attr->getNumDocs()); + EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); + if (_denseTensors) { + TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); + TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4)); + } else { + TEST_DO(assertGetTensor({ {{{"y","1"}}, 11} }, { "x", "y"}, 3)); + TEST_DO(assertGetTensor({}, {"x", "y"}, 4)); + } + TEST_DO(assertGetNoTensor(2)); } -TEST_F("Test compaction of tensor attribute", Fixture("tensor(x{}, y{})")) +void +Fixture::testCompaction() { - f.ensureSpace(4); - Tensor::UP emptytensor = f.createTensor({}, {}); - Tensor::UP emptyxytensor = f.createTensor({}, {"x", "y"}); - Tensor::UP simpletensor = f.createTensor({ {{}, 3} }, { "x", "y"}); - Tensor::UP filltensor = f.createTensor({ {{}, 5} }, { "x", "y"}); - f.setTensor(4, *emptytensor); - f.setTensor(3, *simpletensor); - f.setTensor(2, *filltensor); - f.clearTensor(2); - f.setTensor(2, *filltensor); - search::attribute::Status oldStatus = f.getStatus(); + ensureSpace(4); + Tensor::UP emptytensor = createTensor({}, {}); + Tensor::UP emptyxytensor = createTensor({}, {"x", "y"}); + Tensor::UP simpletensor = createTensor({ {{{"y","1"}}, 11} }, { "x", "y"}); + Tensor::UP filltensor = createTensor({ {{}, 5} }, { "x", "y"}); + setTensor(4, *emptytensor); + setTensor(3, *simpletensor); + setTensor(2, *filltensor); + clearTensor(2); + setTensor(2, *filltensor); + search::attribute::Status oldStatus = getStatus(); search::attribute::Status newStatus = oldStatus; uint64_t iter = 0; uint64_t iterLimit = 100000; for (; iter < iterLimit; ++iter) { - f.clearTensor(2); - f.setTensor(2, *filltensor); - newStatus = f.getStatus(); + clearTensor(2); + setTensor(2, *filltensor); + newStatus = getStatus(); if (newStatus.getUsed() < oldStatus.getUsed()) { break; } @@ -194,16 +322,22 @@ TEST_F("Test compaction of tensor attribute", Fixture("tensor(x{}, y{})")) LOG(info, "iter = %" PRIu64 ", memory usage %" PRIu64 ", -> %" PRIu64, iter, oldStatus.getUsed(), newStatus.getUsed()); - TEST_DO(f.assertGetNoTensor(1)); - TEST_DO(f.assertGetTensor(*filltensor, 2)); - TEST_DO(f.assertGetTensor(*simpletensor, 3)); - TEST_DO(f.assertGetTensor(*emptyxytensor, 4)); + TEST_DO(assertGetNoTensor(1)); + if (_denseTensors) { + emptyxytensor = expEmptyDenseTensor(); + simpletensor = expDenseTensor3(); + filltensor = expDenseFillTensor(); + } + TEST_DO(assertGetTensor(*filltensor, 2)); + TEST_DO(assertGetTensor(*simpletensor, 3)); + TEST_DO(assertGetTensor(*emptyxytensor, 4)); } -TEST_F("Test tensortype file header tag", Fixture("tensor(x[10])")) +void +Fixture::testTensorTypeFileHeaderTag() { - f.ensureSpace(4); - TEST_DO(f.save()); + ensureSpace(4); + TEST_DO(save()); vespalib::FileHeader header; FastOS_File file; @@ -211,15 +345,90 @@ TEST_F("Test tensortype file header tag", Fixture("tensor(x[10])")) (void) header.readFile(file); file.Close(); EXPECT_TRUE(header.hasTag("tensortype")); - EXPECT_EQUAL("tensor(x[10])", header.getTag("tensortype").asString()); + EXPECT_EQUAL(_typeSpec, header.getTag("tensortype").asString()); + if (_useDenseTensorAttribute) { + EXPECT_EQUAL(1u, header.getTag("version").asInteger()); + } else { + EXPECT_EQUAL(0u, header.getTag("version").asInteger()); + } } -TEST_F("Require that tensor attribute can provide empty tensor of correct type", Fixture("tensor(x[10])")) + +void +Fixture::testEmptyTensor() { - const TensorAttribute &tensorAttr = *f._tensorAttr; + const TensorAttribute &tensorAttr = *_tensorAttr; Tensor::UP emptyTensor = tensorAttr.getEmptyTensor(); - EXPECT_EQUAL(emptyTensor->getType(), tensorAttr.getConfig().tensorType()); - EXPECT_EQUAL(emptyTensor->getType(), ValueType::from_spec("tensor(x[10])")); + if (_denseTensors) { + vespalib::string expSpec = expEmptyDenseTensorSpec(); + EXPECT_EQUAL(emptyTensor->getType(), ValueType::from_spec(expSpec)); + } else { + EXPECT_EQUAL(emptyTensor->getType(), tensorAttr.getConfig().tensorType()); + EXPECT_EQUAL(emptyTensor->getType(), ValueType::from_spec(_typeSpec)); + } +} + + +TEST_F("Test empty sparse tensor attribute", Fixture("tensor()")) +{ + f.testEmptyAttribute(); +} + + +template <class MakeFixture> +void testAll(MakeFixture &&f) +{ + TEST_DO(f()->testEmptyAttribute()); + TEST_DO(f()->testSetTensorValue()); + TEST_DO(f()->testSaveLoad()); + TEST_DO(f()->testCompaction()); + TEST_DO(f()->testTensorTypeFileHeaderTag()); + TEST_DO(f()->testEmptyTensor()); +} + +TEST("Test sparse tensors with generic tensor attribute") +{ + testAll([]() { return std::make_shared<Fixture>(sparseSpec); }); +} + +TEST("Test dense tensors with generic tensor attribute") +{ + testAll([]() { return std::make_shared<Fixture>(denseSpec); }); +} + +TEST("Test dense tensors with dense tensor attribute") +{ + testAll([]() { return std::make_shared<Fixture>(denseSpec, true); }); +} + +TEST("Test dense tensors with generic tensor attribute with unbound x and y dims") +{ + testAll([]() { return std::make_shared<Fixture>(denseAbstractSpec_xy); }); +} + +TEST("Test dense tensors with dense tensor attribute with unbound x and y dims") +{ + testAll([]() { return std::make_shared<Fixture>(denseAbstractSpec_xy, true); }); +} + +TEST("Test dense tensors with generic tensor attribute with unbound x dim") +{ + testAll([]() { return std::make_shared<Fixture>(denseAbstractSpec_x); }); +} + +TEST("Test dense tensors with dense tensor attribute with unbound x dim") +{ + testAll([]() { return std::make_shared<Fixture>(denseAbstractSpec_x, true); }); +} + +TEST("Test dense tensors with generic tensor attribute with unbound y dim") +{ + testAll([]() { return std::make_shared<Fixture>(denseAbstractSpec_y); }); +} + +TEST("Test dense tensors with dense tensor attribute with unbound y dim") +{ + testAll([]() { return std::make_shared<Fixture>(denseAbstractSpec_y, true); }); } -TEST_MAIN() { TEST_RUN_ALL(); } +TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); } diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp index 8dee27abcd5..df183b9c319 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp @@ -12,6 +12,7 @@ LOG_SETUP(".createsinglestd"); #include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/tensor/generic_tensor_attribute.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> namespace search { @@ -58,7 +59,11 @@ AttributeFactory::createSingleStd(const vespalib::string & baseFileName, const C ret.reset(new PredicateAttribute(baseFileName, info)); break; case BasicType::TENSOR: - ret.reset(new attribute::GenericTensorAttribute(baseFileName, info)); + if (info.tensorType().is_dense()) { + ret.reset(new attribute::DenseTensorAttribute(baseFileName, info)); + } else { + ret.reset(new attribute::GenericTensorAttribute(baseFileName, info)); + } break; default: break; diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 475873671b1..6fd517f6937 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -1,6 +1,8 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_tensor OBJECT SOURCES + dense_tensor_attribute.cpp + dense_tensor_attribute_saver.cpp dense_tensor_store.cpp generic_tensor_attribute.cpp generic_tensor_store.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp new file mode 100644 index 00000000000..5bbf9e39550 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -0,0 +1,171 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "dense_tensor_attribute.h" +#include <vespa/vespalib/tensor/tensor.h> +#include "dense_tensor_attribute_saver.h" +#include "tensor_attribute.hpp" + +using vespalib::eval::ValueType; +using vespalib::tensor::Tensor; +using vespalib::tensor::TensorMapper; + +namespace search { + +namespace attribute { + +namespace { + +constexpr uint32_t DENSE_TENSOR_ATTRIBUTE_VERSION = 1; +const vespalib::string tensorTypeTag("tensortype"); + +class TensorReader : public AttributeVector::ReaderBase +{ +private: + static constexpr uint8_t tensorIsNotPresent = 0; + static constexpr uint8_t tensorIsPresent = 1; + vespalib::eval::ValueType _tensorType; + uint32_t _numUnboundDims; + size_t _numBoundCells; + std::vector<uint32_t> _unboundDimSizes; +public: + TensorReader(AttributeVector &attr) + : AttributeVector::ReaderBase(attr), + _tensorType(vespalib::eval::ValueType::from_spec(getDatHeader().getTag(tensorTypeTag).asString())), + _numUnboundDims(0), + _numBoundCells(1), + _unboundDimSizes() + { + for (const auto & dim : _tensorType.dimensions()) { + if (dim.is_bound()) { + _numBoundCells *= dim.size; + } else { + ++_numUnboundDims; + } + } + _unboundDimSizes.resize(_numUnboundDims); + } + size_t getNumCells() { + unsigned char detect; + _datFile->ReadBuf(&detect, sizeof(detect)); + if (detect == tensorIsNotPresent) { + return 0u; + } + if (detect != tensorIsPresent) { + abort(); + } + size_t numCells = _numBoundCells; + if (_numUnboundDims != 0) { + _datFile->ReadBuf(&_unboundDimSizes[0], + _numUnboundDims * sizeof(uint32_t)); + for (auto i = 0u; i < _numUnboundDims; ++i) { + assert(_unboundDimSizes[i] != 0u); + numCells *= _unboundDimSizes[i]; + // TODO: sanity check numCells + } + } + return numCells; + } + const vespalib::eval::ValueType &tensorType() const { return _tensorType; } + const std::vector<uint32_t> &getUnboundDimSizes() const { return _unboundDimSizes; } + void readTensor(void *buf, size_t len) { _datFile->ReadBuf(buf, len); } +}; + +} + +DenseTensorAttribute::DenseTensorAttribute(const vespalib::stringref &baseFileName, + const Config &cfg) + : TensorAttribute(baseFileName, cfg, _denseTensorStore), + _denseTensorStore(cfg.tensorType()) +{ +} + + +DenseTensorAttribute::~DenseTensorAttribute() +{ + getGenerationHolder().clearHoldLists(); + _tensorStore.clearHoldLists(); +} + +void +DenseTensorAttribute::setTensor(DocId docId, const Tensor &tensor) +{ + RefType ref = _denseTensorStore.setTensor( + (_tensorMapper ? *_tensorMapper->map(tensor) : tensor)); + setTensorRef(docId, ref); +} + + +std::unique_ptr<Tensor> +DenseTensorAttribute::getTensor(DocId docId) const +{ + RefType ref; + if (docId < getCommittedDocIdLimit()) { + ref = _refVector[docId]; + } + if (!ref.valid()) { + return std::unique_ptr<Tensor>(); + } + return _denseTensorStore.getTensor(ref); +} + +bool +DenseTensorAttribute::onLoad() +{ + TensorReader tensorReader(*this); + if (!tensorReader.hasData()) { + return false; + } + setCreateSerialNum(tensorReader.getCreateSerialNum()); + assert(tensorReader.getVersion() == DENSE_TENSOR_ATTRIBUTE_VERSION); + assert(getConfig().tensorType().to_spec() == + tensorReader.getDatHeader().getTag(tensorTypeTag).asString()); + uint32_t numDocs(tensorReader.getDocIdLimit()); + uint32_t cellSize(_denseTensorStore.getCellSize()); + _refVector.reset(); + _refVector.unsafe_reserve(numDocs); + for (uint32_t lid = 0; lid < numDocs; ++lid) { + size_t numCells = tensorReader.getNumCells(); + if (numCells != 0u) { + const auto &unboundDimSizes = tensorReader.getUnboundDimSizes(); + auto raw = _denseTensorStore.allocRawBuffer(numCells, unboundDimSizes); + size_t rawLen = numCells * cellSize; + tensorReader.readTensor(raw.first, rawLen); + _refVector.push_back(raw.second); + } else { + _refVector.push_back(RefType()); + } + } + setNumDocs(numDocs); + setCommittedDocIdLimit(numDocs); + return true; +} + + +std::unique_ptr<AttributeSaver> +DenseTensorAttribute::onInitSave() +{ + vespalib::GenerationHandler::Guard guard(getGenerationHandler(). + takeGuard()); + return std::make_unique<DenseTensorAttributeSaver> + (std::move(guard), + this->createSaveTargetConfig(), + getRefCopy(), + _denseTensorStore); +} + +void +DenseTensorAttribute::compactWorst() +{ + doCompactWorst<DenseTensorStore::RefType>(); +} + +uint32_t +DenseTensorAttribute::getVersion() const +{ + return DENSE_TENSOR_ATTRIBUTE_VERSION; +} + +} // namespace search::attribute + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h new file mode 100644 index 00000000000..979207adb1f --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "tensor_attribute.h" +#include "dense_tensor_store.h" + +namespace search { + +namespace attribute { + +/** + * Attribute vector class used to store dense tensors for all + * documents in memory. + */ +class DenseTensorAttribute : public TensorAttribute +{ + DenseTensorStore _denseTensorStore; +public: + DenseTensorAttribute(const vespalib::stringref &baseFileName, const Config &cfg); + virtual ~DenseTensorAttribute(); + virtual void setTensor(DocId docId, const Tensor &tensor) override; + virtual std::unique_ptr<Tensor> getTensor(DocId docId) const override; + virtual bool onLoad() override; + virtual std::unique_ptr<AttributeSaver> onInitSave() override; + virtual void compactWorst() override; + virtual uint32_t getVersion() const override; +}; + + +} // namespace search::attribute + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp new file mode 100644 index 00000000000..67610054fe7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "dense_tensor_attribute_saver.h" +#include <vespa/searchlib/util/bufferwriter.h> +#include "dense_tensor_store.h" + +using vespalib::GenerationHandler; +using search::IAttributeSaveTarget; + +namespace search { + +namespace attribute { + +namespace { + +static const uint8_t tensorIsNotPresent = 0; +static const uint8_t tensorIsPresent = 1; + +} + +DenseTensorAttributeSaver:: +DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, + const IAttributeSaveTarget::Config &cfg, + RefCopyVector &&refs, + const DenseTensorStore &tensorStore) + : AttributeSaver(std::move(guard), cfg), + _refs(std::move(refs)), + _tensorStore(tensorStore) +{ +} + + +DenseTensorAttributeSaver::~DenseTensorAttributeSaver() +{ +} + + +bool +DenseTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) +{ + std::unique_ptr<BufferWriter> + datWriter(saveTarget.datWriter().allocBufferWriter()); + const uint32_t unboundDimSizesSize = _tensorStore.unboundDimSizesSize(); + const uint32_t docIdLimit(_refs.size()); + const uint32_t cellSize = _tensorStore.getCellSize(); + for (uint32_t lid = 0; lid < docIdLimit; ++lid) { + auto raw = _tensorStore.getRawBuffer(_refs[lid]); + if (raw) { + datWriter->write(&tensorIsPresent, sizeof(tensorIsPresent)); + size_t numCells = _tensorStore.getNumCells(raw); + size_t rawLen = numCells * cellSize + unboundDimSizesSize; + datWriter->write(static_cast<const char *>(raw) - unboundDimSizesSize, + rawLen); + } else { + datWriter->write(&tensorIsNotPresent, sizeof(tensorIsNotPresent)); + } + } + datWriter->flush(); + return true; +} + + +} // namespace search::attribute + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h new file mode 100644 index 00000000000..a5d04153ade --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/attribute/attributesaver.h> +#include <vespa/searchlib/attribute/iattributesavetarget.h> +#include "tensor_attribute.h" + +namespace search { + +namespace attribute { + +class DenseTensorStore; + +/* + * Class for saving a tensor attribute. + */ +class DenseTensorAttributeSaver : public AttributeSaver +{ +public: + using RefCopyVector = TensorAttribute::RefCopyVector; +private: + RefCopyVector _refs; + const DenseTensorStore &_tensorStore; + using GenerationHandler = vespalib::GenerationHandler; + + virtual bool onSave(IAttributeSaveTarget &saveTarget) override; +public: + DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, + const IAttributeSaveTarget::Config &cfg, + RefCopyVector &&refs, + const DenseTensorStore &tensorStore); + + virtual ~DenseTensorAttributeSaver(); +}; + +} // namespace search::attribute + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index 28009dad188..82e8870d4eb 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -4,6 +4,7 @@ #include "dense_tensor_store.h" #include <vespa/vespalib/tensor/tensor.h> #include <vespa/vespalib/tensor/dense/dense_tensor_view.h> +#include <vespa/vespalib/tensor/dense/mutable_dense_tensor_view.h> #include <vespa/vespalib/tensor/dense/dense_tensor.h> #include <vespa/vespalib/tensor/serialization/typed_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> @@ -15,6 +16,8 @@ using vespalib::tensor::Tensor; using vespalib::tensor::DenseTensor; using vespalib::tensor::DenseTensorView; +using vespalib::tensor::MutableDenseTensorView; +using vespalib::eval::ValueType; using vespalib::ConstArrayRef; namespace search { @@ -22,29 +25,44 @@ namespace attribute { constexpr size_t MIN_BUFFER_CLUSTERS = 1024; -namespace { +DenseTensorStore::BufferType::BufferType() + : btree::BufferType<char>(RefType::align(1), + MIN_BUFFER_CLUSTERS, + RefType::offsetSize() / RefType::align(1)), + _unboundDimSizesSize(0u) +{ +} -size_t -calcCellsSize(const vespalib::eval::ValueType &type) +DenseTensorStore::BufferType::~BufferType() { - size_t cellsSize = 1; - for (const auto &dim : type.dimensions()) { - cellsSize *= dim.size; - } - return cellsSize; } +void +DenseTensorStore::BufferType::cleanHold(void *buffer, uint64_t offset, + uint64_t len) +{ + // Clear both tensor dimension size information and cells. + memset(static_cast<char *>(buffer) + offset - _unboundDimSizesSize, 0, len); } + DenseTensorStore::DenseTensorStore(const ValueType &type) : TensorStore(_concreteStore), _concreteStore(), - _bufferType(RefType::align(1), - MIN_BUFFER_CLUSTERS, - RefType::offsetSize() / RefType::align(1)), + _bufferType(), _type(type), - _numCells(calcCellsSize(_type)) + _numBoundCells(1u), + _numUnboundDims(0u), + _cellSize(sizeof(double)) { + for (const auto & dim : _type.dimensions()) { + if (dim.is_bound()) { + _numBoundCells *= dim.size; + } else { + ++_numUnboundDims; + } + } + _bufferType.setUnboundDimSizesSize(_numUnboundDims * sizeof(uint32_t)); _store.addType(&_bufferType); _store.initActiveBuffers(); } @@ -54,33 +72,78 @@ DenseTensorStore::~DenseTensorStore() _store.dropBuffers(); } -const double * +const void * DenseTensorStore::getRawBuffer(RefType ref) const { if (!ref.valid()) { return nullptr; } - return _store.getBufferEntry<double>(ref.bufferId(), + return _store.getBufferEntry<char>(ref.bufferId(), ref.offset()); } -std::pair<double *, DenseTensorStore::RefType> -DenseTensorStore::allocRawBuffer() + +size_t +DenseTensorStore::getNumCells(const void *buffer) const { - size_t bufSize = RefType::align(_numCells); - _store.ensureBufferCapacity(_typeId, bufSize); + const uint32_t *unboundDimSizeEnd = static_cast<const uint32_t *>(buffer); + const uint32_t *unboundDimSizeStart = unboundDimSizeEnd - _numUnboundDims; + size_t numCells = _numBoundCells; + for (auto unboundDimSize = unboundDimSizeStart; unboundDimSize != unboundDimSizeEnd; ++unboundDimSize) { + numCells *= *unboundDimSize; + } + return numCells; +} + +namespace { + +void allocateSpaceForFirstUnboundDimSizesInBuffer(char *&buffer, size_t &oldSize, btree::BufferState &state, size_t alignedUnboundDimSizesSize) { + memset(buffer, 0, alignedUnboundDimSizesSize); + state.pushed_back(alignedUnboundDimSizesSize); + state._deadElems += alignedUnboundDimSizesSize; + buffer += alignedUnboundDimSizesSize; + oldSize += alignedUnboundDimSizesSize; +} + +void clearPadAreaAfterBuffer(char *buffer, size_t bufSize, size_t alignedBufSize, uint32_t unboundDimSizesSize) { + size_t padSize = alignedBufSize - unboundDimSizesSize - bufSize; + memset(buffer + bufSize, 0, padSize); +} + +} + +std::pair<void *, DenseTensorStore::RefType> +DenseTensorStore::allocRawBuffer(size_t numCells) +{ + size_t alignedUnboundDimSizesSize = RefType::align(unboundDimSizesSize()); + size_t bufSize = numCells * _cellSize; + size_t alignedBufSize = alignedSize(numCells); + size_t ensureSize = alignedBufSize + alignedUnboundDimSizesSize; + _store.ensureBufferCapacity(_typeId, ensureSize); uint32_t activeBufferId = _store.getActiveBufferId(_typeId); btree::BufferState &state = _store.getBufferState(activeBufferId); size_t oldSize = state.size(); - double *bufferEntryWritePtr = - _store.getBufferEntry<double>(activeBufferId, oldSize); - double *padWritePtr = bufferEntryWritePtr + _numCells; - for (size_t i = _numCells; i < bufSize; ++i) { - *padWritePtr++ = 0.0; + char *buffer = _store.getBufferEntry<char>(activeBufferId, oldSize); + if (oldSize <= alignedUnboundDimSizesSize) { + allocateSpaceForFirstUnboundDimSizesInBuffer(buffer, oldSize, state, alignedUnboundDimSizesSize); + } + clearPadAreaAfterBuffer(buffer, bufSize, alignedBufSize, unboundDimSizesSize()); + state.pushed_back(alignedBufSize); + return std::make_pair(buffer, RefType(oldSize, activeBufferId)); +} + +std::pair<void *, DenseTensorStore::RefType> +DenseTensorStore::allocRawBuffer(size_t numCells, + const std::vector<uint32_t> &unboundDimSizes) +{ + assert(unboundDimSizes.size() == _numUnboundDims); + auto ret = allocRawBuffer(numCells); + if (_numUnboundDims > 0) { + memcpy(static_cast<char *>(ret.first) - unboundDimSizesSize(), + &unboundDimSizes[0], unboundDimSizesSize()); } - state.pushed_back(bufSize); - return std::make_pair(bufferEntryWritePtr, - RefType(oldSize, activeBufferId)); + assert(numCells == getNumCells(ret.first)); + return ret; } void @@ -89,7 +152,9 @@ DenseTensorStore::holdTensor(EntryRef ref) if (!ref.valid()) { return; } - _concreteStore.holdElem(ref, _numCells); + const void *buffer = getRawBuffer(ref); + size_t numCells = getNumCells(buffer); + _concreteStore.holdElem(ref, alignedSize(numCells)); } TensorStore::EntryRef @@ -98,12 +163,37 @@ DenseTensorStore::move(EntryRef ref) { return RefType(); } auto oldraw = getRawBuffer(ref); - auto newraw = allocRawBuffer(); - memcpy(newraw.first, oldraw, _numCells * sizeof(double)); - _concreteStore.holdElem(ref, _numCells); + size_t numCells = getNumCells(oldraw); + auto newraw = allocRawBuffer(numCells); + memcpy(static_cast<char *>(newraw.first) - unboundDimSizesSize(), + static_cast<const char *>(oldraw) - unboundDimSizesSize(), + numCells * _cellSize + unboundDimSizesSize()); + _concreteStore.holdElem(ref, alignedSize(numCells)); return newraw.second; } +namespace { + +ValueType makeConcreteType(const ValueType &type, + const void *buffer, + uint32_t numUnboundDims) +{ + std::vector<ValueType::Dimension> dimensions(type.dimensions()); + const uint32_t *unboundDimSizeEnd = static_cast<const uint32_t *>(buffer); + const uint32_t *unboundDimSize = unboundDimSizeEnd - numUnboundDims; + for (auto &dim : dimensions) { + if (!dim.is_bound()) { + assert(unboundDimSize != unboundDimSizeEnd); + dim.size = *unboundDimSize; + ++unboundDimSize; + } + } + assert(unboundDimSize == unboundDimSizeEnd); + return ValueType::tensor_type(std::move(dimensions)); +} + +} + std::unique_ptr<Tensor> DenseTensorStore::getTensor(EntryRef ref) const { @@ -111,17 +201,69 @@ DenseTensorStore::getTensor(EntryRef ref) const if (raw == nullptr) { return std::unique_ptr<Tensor>(); } - return std::make_unique<DenseTensorView>(_type, ConstArrayRef<double>(raw, _numCells)); + size_t numCells = getNumCells(raw); + if (_numUnboundDims == 0) { + return std::make_unique<DenseTensorView> + (_type, + ConstArrayRef<double>(static_cast<const double *>(raw), numCells)); + } + return std::make_unique<MutableDenseTensorView> + (makeConcreteType(_type, raw, _numUnboundDims), + ConstArrayRef<double>(static_cast<const double *>(raw), numCells)); +} + +namespace +{ + +void +checkMatchingType(const ValueType &lhs, const ValueType &rhs, size_t numCells) +{ + size_t checkNumCells = 1u; + auto rhsItr = rhs.dimensions().cbegin(); + auto rhsItrEnd = rhs.dimensions().cend(); + for (const auto &dim : lhs.dimensions()) { + assert(rhsItr != rhsItrEnd); + assert(dim.name == rhsItr->name); + assert(rhsItr->is_bound()); + assert(!dim.is_bound() || dim.size == rhsItr->size); + checkNumCells *= rhsItr->size; + ++rhsItr; + } + assert(numCells == checkNumCells); + assert(rhsItr == rhsItrEnd); +} + +void +setDenseTensorUnboundDimSizes(void *buffer, const ValueType &lhs, uint32_t numUnboundDims, const ValueType &rhs) +{ + uint32_t *unboundDimSizeEnd = static_cast<uint32_t *>(buffer); + uint32_t *unboundDimSize = unboundDimSizeEnd - numUnboundDims; + auto rhsItr = rhs.dimensions().cbegin(); + auto rhsItrEnd = rhs.dimensions().cend(); + for (const auto &dim : lhs.dimensions()) { + assert (rhsItr != rhsItrEnd); + if (!dim.is_bound()) { + assert(unboundDimSize != unboundDimSizeEnd); + *unboundDimSize = rhsItr->size; + ++unboundDimSize; + } + ++rhsItr; + } + assert (rhsItr == rhsItrEnd); + assert(unboundDimSize == unboundDimSizeEnd); +} + } template <class TensorType> TensorStore::EntryRef DenseTensorStore::setDenseTensor(const TensorType &tensor) { - assert(tensor.type() == _type); - assert(tensor.cells().size() == _numCells); - auto raw = allocRawBuffer(); - memcpy(raw.first, &tensor.cells()[0], _numCells * sizeof(double)); + size_t numCells = tensor.cells().size(); + checkMatchingType(_type, tensor.type(), numCells); + auto raw = allocRawBuffer(numCells); + setDenseTensorUnboundDimSizes(raw.first, _type, _numUnboundDims, tensor.type()); + memcpy(raw.first, &tensor.cells()[0], numCells * _cellSize); return raw.second; } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 062e1854dff..6c507289c03 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -12,30 +12,65 @@ namespace attribute { /** * Class for storing dense tensors with known bounds in memory, used * by DenseTensorAttribute. + * + * Tensor dimension size information for unbound dimensions is at + * negative offset to preserve cell array aligment without + * introducing excessive padding, e.g. if tensor store is setup for + * tensors of type tensor(x[]) then a tensor of type tensor(x[3]) will + * use 32 bytes (inclusive 4 bytes padding). + * + * If both start of tensor dimension size information and start of + * tensor cells were to be 32 byte aligned then tensors of type tensor(x[3]) + * would use 64 bytes. */ class DenseTensorStore : public TensorStore { public: - // 2 entry alignment, entry type is double => 16 bytes alignment - using RefType = btree::AlignedEntryRefT<22, 1>; + // 32 entry alignment, entry type is char => 32 bytes alignment + using RefType = btree::AlignedEntryRefT<22, 5>; using DataStoreType = btree::DataStoreT<RefType>; using ValueType = vespalib::eval::ValueType; + + class BufferType : public btree::BufferType<char> + { + uint32_t _unboundDimSizesSize; + public: + BufferType(); + virtual ~BufferType(); + virtual void + cleanHold(void *buffer, uint64_t offset, uint64_t len) override; + uint32_t unboundDimSizesSize() const { return _unboundDimSizesSize; } + void setUnboundDimSizesSize(uint32_t unboundDimSizesSize_in) { + _unboundDimSizesSize = unboundDimSizesSize_in; + } + }; private: DataStoreType _concreteStore; - btree::BufferType<double> _bufferType; + BufferType _bufferType; ValueType _type; // type of dense tensor - size_t _numCells; // number of cells in dense tensor + size_t _numBoundCells; // product of bound dimension sizes + uint32_t _numUnboundDims; + uint32_t _cellSize; // size of a cell (e.g. double => 8) + + size_t unboundCells(const void *buffer) const; template <class TensorType> TensorStore::EntryRef setDenseTensor(const TensorType &tensor); + std::pair<void *, RefType> allocRawBuffer(size_t numCells); + size_t alignedSize(size_t numCells) const { + return RefType::align(numCells * _cellSize + unboundDimSizesSize()); + } public: + uint32_t unboundDimSizesSize() const { return _bufferType.unboundDimSizesSize(); } DenseTensorStore(const ValueType &type); virtual ~DenseTensorStore(); - size_t numCells() const { return _numCells; } - const double *getRawBuffer(RefType ref) const; - std::pair<double *, RefType> allocRawBuffer(); + size_t getNumCells(const void *buffer) const; + uint32_t getCellSize() const { return _cellSize; } + const void *getRawBuffer(RefType ref) const; + std::pair<void *, RefType> + allocRawBuffer(size_t numCells, const std::vector<uint32_t> &unboundDimSizes); virtual void holdTensor(EntryRef ref) override; virtual EntryRef move(EntryRef ref) override; std::unique_ptr<Tensor> getTensor(EntryRef ref) const; diff --git a/searchlib/src/vespa/searchlib/util/fileheadertk.cpp b/searchlib/src/vespa/searchlib/util/fileheadertk.cpp index d1ae39d15c7..6451df32b69 100644 --- a/searchlib/src/vespa/searchlib/util/fileheadertk.cpp +++ b/searchlib/src/vespa/searchlib/util/fileheadertk.cpp @@ -1,23 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> +#include <vespa/vespalib/component/vtag.h> #include "fileheadertk.h" using namespace search; using vespalib::GenericHeader; +using namespace vespalib; void FileHeaderTk::addVersionTags(vespalib::GenericHeader &header) { -#ifdef V_TAG - header.putTag(GenericHeader::Tag("version-tag", V_TAG));; - header.putTag(GenericHeader::Tag("version-date", V_TAG_DATE));; - header.putTag(GenericHeader::Tag("version-pkg", V_TAG_PKG));; - header.putTag(GenericHeader::Tag("version-arch", V_TAG_ARCH));; - header.putTag(GenericHeader::Tag("version-system", V_TAG_SYSTEM)); - header.putTag(GenericHeader::Tag("version-system-rev", V_TAG_SYSTEM_REV)); - header.putTag(GenericHeader::Tag("version-builder", V_TAG_BUILDER)); - header.putTag(GenericHeader::Tag("version-component", V_TAG_COMPONENT)); -#else - (void)header; -#endif + header.putTag(GenericHeader::Tag("version-tag", VersionTag)); + header.putTag(GenericHeader::Tag("version-date", VersionTagDate)); + header.putTag(GenericHeader::Tag("version-pkg", VersionTagPkg)); + header.putTag(GenericHeader::Tag("version-arch", VersionTagArch)); + header.putTag(GenericHeader::Tag("version-system", VersionTagSystem)); + header.putTag(GenericHeader::Tag("version-system-rev", VersionTagSystemRev)); + header.putTag(GenericHeader::Tag("version-builder", VersionTagBuilder)); + header.putTag(GenericHeader::Tag("version-component", VersionTagComponent)); } diff --git a/slobrok/src/vespa/slobrok/server/CMakeLists.txt b/slobrok/src/vespa/slobrok/server/CMakeLists.txt index da337624ce3..2ac6b28a8b2 100644 --- a/slobrok/src/vespa/slobrok/server/CMakeLists.txt +++ b/slobrok/src/vespa/slobrok/server/CMakeLists.txt @@ -15,11 +15,11 @@ vespa_add_library(slobrok_slobrokserver rpc_server_manager.cpp rpc_server_map.cpp rpchooks.cpp + rpcmirror.cpp sbenv.cpp selfcheck.cpp slobrokserver.cpp visible_map.cpp - vtag.cpp metrics_producer.cpp INSTALL lib64 DEPENDS diff --git a/slobrok/src/vespa/slobrok/server/named_service.h b/slobrok/src/vespa/slobrok/server/named_service.h index f5221b75657..9d04475573a 100644 --- a/slobrok/src/vespa/slobrok/server/named_service.h +++ b/slobrok/src/vespa/slobrok/server/named_service.h @@ -18,17 +18,15 @@ namespace slobrok { class NamedService { -private: - NamedService(const NamedService &); // Not used - NamedService &operator=(const NamedService &); // Not used - protected: std::string _name; std::string _spec; public: - NamedService(const char *name, - const char *spec); + NamedService(const NamedService &) = delete; + NamedService &operator=(const NamedService &) = delete; + + NamedService(const char *name, const char *spec); virtual ~NamedService(); const char *getName() const { return _name.c_str(); } diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp index 5f610ae2e45..5e7f2c401bd 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp +++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp @@ -1,12 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include "vtag.h" - -#include <vespa/log/log.h> -LOG_SETUP(".rpchooks"); - +#include <vespa/vespalib/component/vtag.h> #include <vespa/fnet/frt/frt.h> - #include "rpchooks.h" #include "ok_state.h" #include "named_service.h" @@ -17,6 +12,10 @@ LOG_SETUP(".rpchooks"); #include "visible_map.h" #include "rpcmirror.h" +#include <vespa/log/log.h> + +LOG_SETUP(".rpchooks"); + namespace slobrok { namespace { @@ -558,9 +557,9 @@ RPCHooks::rpc_version(FRT_RPCRequest *req) _cnts.adminReqs++; std::string ver; - char *s = VersionTag; + char *s = vespalib::VersionTag; bool needdate = true; - if (strncmp(VersionTag, "V_", 2) == 0) { + if (strncmp(vespalib::VersionTag, "V_", 2) == 0) { s += 2; do { while (strchr("0123456789", *s) != NULL) { @@ -596,7 +595,7 @@ RPCHooks::rpc_version(FRT_RPCRequest *req) } if (needdate) { ver.append("-"); - s = VersionTagDate; + s = vespalib::VersionTagDate; char *e = strchr(s, '-'); if (e == NULL) { ver.append(s); diff --git a/slobrok/src/vespa/slobrok/server/rpcmirror.cpp b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp new file mode 100644 index 00000000000..12cae8ad500 --- /dev/null +++ b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "rpcmirror.h" +#include <vespa/log/log.h> + +LOG_SETUP(".rpcmirror"); + +namespace slobrok { + +MirrorFetch::MirrorFetch(FRT_Supervisor *orb, + FRT_RPCRequest *req, + VisibleMap &map, + vespalib::GenCnt gen) + : FNET_Task(orb->GetScheduler()), + _req(req), + _map(map), + _gen(gen) +{ +} + + +MirrorFetch::~MirrorFetch() +{ + LOG_ABORT("Should never be called!"); +} + + +void +MirrorFetch::completeReq() +{ + vespalib::GenCnt newgen = _map.genCnt(); + if (newgen == _gen) { // no change + _req->GetReturn()->AddStringArray(0); + _req->GetReturn()->AddStringArray(0); + } else { + std::vector<const NamedService *> rpcsrvlist = _map.allVisible(); + + FRT_Values &dst = *_req->GetReturn(); + size_t sz = rpcsrvlist.size(); + FRT_StringValue *names = dst.AddStringArray(sz); + FRT_StringValue *specs = dst.AddStringArray(sz); + for (uint32_t i = 0; i < rpcsrvlist.size(); ++i) { + dst.SetString(&names[i], rpcsrvlist[i]->getName()); + dst.SetString(&specs[i], rpcsrvlist[i]->getSpec()); + } + if (sz > 0) { + LOG(debug, "mirrorFetch %p -> %u, last [%s,%s]", + this, + (unsigned int)sz, + dst[0]._string_array._pt[sz-1]._str, + dst[1]._string_array._pt[sz-1]._str); + } else { + LOG(debug, "mirrorFetch %p -> 0 size", this); + } + } + _req->GetReturn()->AddInt32(newgen.getAsInt()); + LOG(debug, "mirrorFetch %p done (gen %d -> gen %d)", + this, _gen.getAsInt(), newgen.getAsInt()); + _req->Return(); +} + + +void +MirrorFetch::PerformTask() +{ + // cancel update notification + _map.removeUpdateListener(this); + completeReq(); +} + + +void +MirrorFetch::updated(VisibleMap &map) +{ + LOG_ASSERT(&map == &_map); + (void) ↦ + // unschedule timeout task + Unschedule(); + completeReq(); +} + + +void +MirrorFetch::aborted(VisibleMap &map) +{ + LOG_ASSERT(&map == &_map); + (void) ↦ + // unschedule timeout task + Unschedule(); + _req->SetError(FRTE_RPC_METHOD_FAILED, "slobrok shutting down"); + _req->Return(); +} + + +void +MirrorFetch::invoke(uint32_t msTimeout) +{ + _req->Detach(); + LOG(debug, "MirrorFetch %p invoked from %s (gen %d, timeout %d ms)", + this, _req->GetConnection()->GetSpec(), _gen.getAsInt(), msTimeout); + if (_map.genCnt() != _gen || msTimeout == 0) { + completeReq(); + } else { + _map.addUpdateListener(this); // register as update listener + if (msTimeout > 10000) + msTimeout = 10000; + Schedule((double) msTimeout / 1000.0); + } +} + +IncrementalFetch::IncrementalFetch(FRT_Supervisor *orb, + FRT_RPCRequest *req, + VisibleMap &map, + vespalib::GenCnt gen) + : FNET_Task(orb->GetScheduler()), + _req(req), + _map(map), + _gen(gen) +{ +} + + +IncrementalFetch::~IncrementalFetch() +{ + LOG_ABORT("Should never be called!"); +} + + +void +IncrementalFetch::completeReq() +{ + vespalib::GenCnt newgen = _map.genCnt(); + VisibleMap::MapDiff diff; + FRT_Values &dst = *_req->GetReturn(); + + if (newgen == _gen) { // no change + dst.AddInt32(_gen.getAsInt()); + } else if (_map.hasHistory(_gen)) { + diff = _map.history(_gen); + dst.AddInt32(_gen.getAsInt()); + } else { + dst.AddInt32(0); + diff.updated = _map.allVisible(); + } + + size_t sz = diff.removed.size(); + FRT_StringValue *rem = dst.AddStringArray(sz); + for (uint32_t i = 0; i < sz; ++i) { + dst.SetString(&rem[i], diff.removed[i].c_str()); + } + + sz = diff.updated.size(); + FRT_StringValue *names = dst.AddStringArray(sz); + FRT_StringValue *specs = dst.AddStringArray(sz); + for (uint32_t i = 0; i < sz; ++i) { + dst.SetString(&names[i], diff.updated[i]->getName()); + dst.SetString(&specs[i], diff.updated[i]->getSpec()); + } + + dst.AddInt32(newgen.getAsInt()); + LOG(debug, "mirrorFetch %p done (gen %d -> gen %d)", + this, _gen.getAsInt(), newgen.getAsInt()); + _req->Return(); +} + + +void +IncrementalFetch::PerformTask() +{ + // cancel update notification + _map.removeUpdateListener(this); + completeReq(); +} + + +void +IncrementalFetch::updated(VisibleMap &map) +{ + LOG_ASSERT(&map == &_map); + (void) ↦ + // unschedule timeout task + Unschedule(); + completeReq(); +} + + +void +IncrementalFetch::aborted(VisibleMap &map) +{ + LOG_ASSERT(&map == &_map); + (void) ↦ + // unschedule timeout task + Unschedule(); + _req->SetError(FRTE_RPC_METHOD_FAILED, "slobrok shutting down"); + _req->Return(); +} + + +void +IncrementalFetch::invoke(uint32_t msTimeout) +{ + _req->Detach(); + LOG(debug, "IncrementalFetch %p invoked from %s (gen %d, timeout %d ms)", + this, _req->GetConnection()->GetSpec(), _gen.getAsInt(), msTimeout); + if (_map.genCnt() != _gen || msTimeout == 0) { + completeReq(); + } else { + _map.addUpdateListener(this); // register as update listener + if (msTimeout > 10000) + msTimeout = 10000; + Schedule((double) msTimeout / 1000.0); + } +} + +} // namespace slobrok + diff --git a/slobrok/src/vespa/slobrok/server/rpcmirror.h b/slobrok/src/vespa/slobrok/server/rpcmirror.h index 889b00c6ba1..1bb31720db5 100644 --- a/slobrok/src/vespa/slobrok/server/rpcmirror.h +++ b/slobrok/src/vespa/slobrok/server/rpcmirror.h @@ -1,273 +1,53 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -namespace slobrok { +#include "visible_map.h" -namespace { +namespace slobrok { class MirrorFetch : public FNET_Task, public VisibleMap::IUpdateListener { private: - MirrorFetch(const MirrorFetch &); - MirrorFetch& operator=(const MirrorFetch &); - FRT_RPCRequest *_req; VisibleMap &_map; vespalib::GenCnt _gen; public: - MirrorFetch(FRT_Supervisor *orb, - FRT_RPCRequest *req, - VisibleMap &map, - vespalib::GenCnt gen); - virtual ~MirrorFetch(); + MirrorFetch(const MirrorFetch &) = delete; + MirrorFetch& operator=(const MirrorFetch &) = delete; + + MirrorFetch(FRT_Supervisor *orb, FRT_RPCRequest *req, VisibleMap &map, vespalib::GenCnt gen); + ~MirrorFetch(); void completeReq(); - virtual void PerformTask(); - virtual void updated(VisibleMap &map); - virtual void aborted(VisibleMap &map); + void PerformTask() override; + void updated(VisibleMap &map) override; + void aborted(VisibleMap &map) override; void invoke(uint32_t msTimeout); }; - -MirrorFetch::MirrorFetch(FRT_Supervisor *orb, - FRT_RPCRequest *req, - VisibleMap &map, - vespalib::GenCnt gen) - : FNET_Task(orb->GetScheduler()), - _req(req), - _map(map), - _gen(gen) -{ -} - - -MirrorFetch::~MirrorFetch() -{ - LOG_ABORT("Should never be called!"); -} - - -void -MirrorFetch::completeReq() -{ - vespalib::GenCnt newgen = _map.genCnt(); - if (newgen == _gen) { // no change - _req->GetReturn()->AddStringArray(0); - _req->GetReturn()->AddStringArray(0); - } else { - std::vector<const NamedService *> rpcsrvlist = _map.allVisible(); - - FRT_Values &dst = *_req->GetReturn(); - size_t sz = rpcsrvlist.size(); - FRT_StringValue *names = dst.AddStringArray(sz); - FRT_StringValue *specs = dst.AddStringArray(sz); - for (uint32_t i = 0; i < rpcsrvlist.size(); ++i) { - dst.SetString(&names[i], rpcsrvlist[i]->getName()); - dst.SetString(&specs[i], rpcsrvlist[i]->getSpec()); - } - if (sz > 0) { - LOG(debug, "mirrorFetch %p -> %u, last [%s,%s]", - this, - (unsigned int)sz, - dst[0]._string_array._pt[sz-1]._str, - dst[1]._string_array._pt[sz-1]._str); - } else { - LOG(debug, "mirrorFetch %p -> 0 size", this); - } - } - _req->GetReturn()->AddInt32(newgen.getAsInt()); - LOG(debug, "mirrorFetch %p done (gen %d -> gen %d)", - this, _gen.getAsInt(), newgen.getAsInt()); - _req->Return(); -} - - -void -MirrorFetch::PerformTask() -{ - // cancel update notification - _map.removeUpdateListener(this); - completeReq(); -} - - -void -MirrorFetch::updated(VisibleMap &map) -{ - LOG_ASSERT(&map == &_map); - (void) ↦ - // unschedule timeout task - Unschedule(); - completeReq(); -} - - -void -MirrorFetch::aborted(VisibleMap &map) -{ - LOG_ASSERT(&map == &_map); - (void) ↦ - // unschedule timeout task - Unschedule(); - _req->SetError(FRTE_RPC_METHOD_FAILED, "slobrok shutting down"); - _req->Return(); -} - - -void -MirrorFetch::invoke(uint32_t msTimeout) -{ - _req->Detach(); - LOG(debug, "MirrorFetch %p invoked from %s (gen %d, timeout %d ms)", - this, _req->GetConnection()->GetSpec(), _gen.getAsInt(), msTimeout); - if (_map.genCnt() != _gen || msTimeout == 0) { - completeReq(); - } else { - _map.addUpdateListener(this); // register as update listener - if (msTimeout > 10000) - msTimeout = 10000; - Schedule((double) msTimeout / 1000.0); - } -} - - - class IncrementalFetch : public FNET_Task, public VisibleMap::IUpdateListener { private: - IncrementalFetch(const IncrementalFetch &); - IncrementalFetch& operator=(const IncrementalFetch &); - FRT_RPCRequest *_req; VisibleMap &_map; vespalib::GenCnt _gen; public: - IncrementalFetch(FRT_Supervisor *orb, - FRT_RPCRequest *req, - VisibleMap &map, - vespalib::GenCnt gen); - virtual ~IncrementalFetch(); + IncrementalFetch(const IncrementalFetch &) = delete; + IncrementalFetch& operator=(const IncrementalFetch &) = delete; + + IncrementalFetch(FRT_Supervisor *orb, FRT_RPCRequest *req, VisibleMap &map, vespalib::GenCnt gen); + ~IncrementalFetch(); void completeReq(); - virtual void PerformTask(); - virtual void updated(VisibleMap &map); - virtual void aborted(VisibleMap &map); + void PerformTask() override; + void updated(VisibleMap &map) override; + void aborted(VisibleMap &map) override; void invoke(uint32_t msTimeout); }; - -IncrementalFetch::IncrementalFetch(FRT_Supervisor *orb, - FRT_RPCRequest *req, - VisibleMap &map, - vespalib::GenCnt gen) - : FNET_Task(orb->GetScheduler()), - _req(req), - _map(map), - _gen(gen) -{ -} - - -IncrementalFetch::~IncrementalFetch() -{ - LOG_ABORT("Should never be called!"); -} - - -void -IncrementalFetch::completeReq() -{ - vespalib::GenCnt newgen = _map.genCnt(); - VisibleMap::MapDiff diff; - FRT_Values &dst = *_req->GetReturn(); - - if (newgen == _gen) { // no change - dst.AddInt32(_gen.getAsInt()); - } else if (_map.hasHistory(_gen)) { - diff = _map.history(_gen); - dst.AddInt32(_gen.getAsInt()); - } else { - dst.AddInt32(0); - diff.updated = _map.allVisible(); - } - - size_t sz = diff.removed.size(); - FRT_StringValue *rem = dst.AddStringArray(sz); - for (uint32_t i = 0; i < sz; ++i) { - dst.SetString(&rem[i], diff.removed[i].c_str()); - } - - sz = diff.updated.size(); - FRT_StringValue *names = dst.AddStringArray(sz); - FRT_StringValue *specs = dst.AddStringArray(sz); - for (uint32_t i = 0; i < sz; ++i) { - dst.SetString(&names[i], diff.updated[i]->getName()); - dst.SetString(&specs[i], diff.updated[i]->getSpec()); - } - - dst.AddInt32(newgen.getAsInt()); - LOG(debug, "mirrorFetch %p done (gen %d -> gen %d)", - this, _gen.getAsInt(), newgen.getAsInt()); - _req->Return(); -} - - -void -IncrementalFetch::PerformTask() -{ - // cancel update notification - _map.removeUpdateListener(this); - completeReq(); -} - - -void -IncrementalFetch::updated(VisibleMap &map) -{ - LOG_ASSERT(&map == &_map); - (void) ↦ - // unschedule timeout task - Unschedule(); - completeReq(); -} - - -void -IncrementalFetch::aborted(VisibleMap &map) -{ - LOG_ASSERT(&map == &_map); - (void) ↦ - // unschedule timeout task - Unschedule(); - _req->SetError(FRTE_RPC_METHOD_FAILED, "slobrok shutting down"); - _req->Return(); -} - - -void -IncrementalFetch::invoke(uint32_t msTimeout) -{ - _req->Detach(); - LOG(debug, "IncrementalFetch %p invoked from %s (gen %d, timeout %d ms)", - this, _req->GetConnection()->GetSpec(), _gen.getAsInt(), msTimeout); - if (_map.genCnt() != _gen || msTimeout == 0) { - completeReq(); - } else { - _map.addUpdateListener(this); // register as update listener - if (msTimeout > 10000) - msTimeout = 10000; - Schedule((double) msTimeout / 1000.0); - } -} - - -} // namespace <unnamed> - -//----------------------------------------------------------------------------- - } // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/visible_map.h b/slobrok/src/vespa/slobrok/server/visible_map.h index 04a61ea1b4d..d67cb9ace70 100644 --- a/slobrok/src/vespa/slobrok/server/visible_map.h +++ b/slobrok/src/vespa/slobrok/server/visible_map.h @@ -8,11 +8,10 @@ #include <string> #include "history.h" +#include "named_service.h" namespace slobrok { -class NamedService; - /** * @class VisibleMap * @brief API to the collection of NamedService diff --git a/slobrok/src/vespa/slobrok/server/vtag.cpp b/slobrok/src/vespa/slobrok/server/vtag.cpp deleted file mode 100644 index 380d653bd28..00000000000 --- a/slobrok/src/vespa/slobrok/server/vtag.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "vtag.h" - -#ifndef V_TAG -#define V_TAG "NOTAG" -#define V_TAG_DATE "NOTAG" -#define V_TAG_SYSTEM "NOTAG" -#define V_TAG_SYSTEM_REV "NOTAG" -#define V_TAG_BUILDER "NOTAG" -#endif - -namespace slobrok { - -char VersionTag[] = V_TAG; -char VersionTagDate[] = V_TAG_DATE; -char VersionTagSystem[] = V_TAG_SYSTEM; -char VersionTagSystemRev[] = V_TAG_SYSTEM_REV; -char VersionTagBuilder[] = V_TAG_BUILDER; - -} // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/vtag.h b/slobrok/src/vespa/slobrok/server/vtag.h deleted file mode 100644 index 6e9227a5951..00000000000 --- a/slobrok/src/vespa/slobrok/server/vtag.h +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace slobrok { - -extern char VersionTag[]; -extern char VersionTagType[]; -extern char VersionTagValue[]; -extern char VersionTagDate[]; -extern char VersionTagSystem[]; -extern char VersionTagSystemRev[]; -extern char VersionTagBuilder[]; - -} // namespace slobrok - diff --git a/storage/src/vespa/storage/common/CMakeLists.txt b/storage/src/vespa/storage/common/CMakeLists.txt index e699f055d02..3b9eb376eb7 100644 --- a/storage/src/vespa/storage/common/CMakeLists.txt +++ b/storage/src/vespa/storage/common/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_library(storage_common OBJECT statusmetricconsumer.cpp storagelink.cpp storagelinkqueued.cpp - vtag.cpp bucketoperationlogger.cpp messagebucketid.cpp messagesender.cpp diff --git a/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp b/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp index 110df2c2b12..670f4ef6369 100644 --- a/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp +++ b/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> #include "versionreporter.h" -#include <vespa/storage/common/vtag.h> -#include <vespa/vespalib/component/version.h> +#include <vespa/vespalib/component/vtag.h> namespace storage { @@ -12,7 +11,7 @@ using End = vespalib::JsonStream::End; } void VersionReporter::report(vespalib::JsonStream& jsonreport) { jsonreport << "vtag" << Object() - << "version" << Vtag::currentVersion.toString() + << "version" << vespalib::Vtag::currentVersion.toString() << End(); } diff --git a/storage/src/vespa/storage/common/vtag.cpp b/storage/src/vespa/storage/common/vtag.cpp deleted file mode 100644 index cf352006904..00000000000 --- a/storage/src/vespa/storage/common/vtag.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <string.h> -#include <stdio.h> -#include <vespa/storage/common/vtag.h> -#include <vespa/vespalib/component/version.h> - -#ifndef V_TAG -#define V_TAG "NOTAG" -#define V_TAG_DATE "NOTAG" -#define V_TAG_SYSTEM "NOTAG" -#define V_TAG_SYSTEM_REV "NOTAG" -#define V_TAG_BUILDER "NOTAG" -#define V_TAG_VERSION "0" -#define V_TAG_ARCH "NOTAG" -#endif - -namespace storage { - -char VersionTag[] = V_TAG; -char VersionTagDate[] = V_TAG_DATE; -char VersionTagSystem[] = V_TAG_SYSTEM; -char VersionTagSystemRev[] = V_TAG_SYSTEM_REV; -char VersionTagBuilder[] = V_TAG_BUILDER; -char VersionTagPkg[] = V_TAG_PKG; -char VersionTagComponent[] = V_TAG_COMPONENT; -char VersionTagArch[] = V_TAG_ARCH; - -vespalib::Version Vtag::currentVersion(VersionTagComponent); - -void -Vtag::printVersionNice() -{ - char *s = VersionTag; - bool needdate = true; - if (strncmp(VersionTag, "V_", 2) == 0) { - s += 2; - do { - while (strchr("0123456789", *s) != NULL) { - printf("%c", *s++); - } - if (strncmp(s, "_RELEASE", 8) == 0) { - needdate = false; - break; - } - if (strncmp(s, "_RC", 3) == 0) { - char *e = strchr(s, '-'); - if (e == NULL) { - printf("%s", s); - } else { - printf("%.*s", (int)(e-s), s); - } - needdate = false; - break; - } - if (*s == '_' && strchr("0123456789", *++s)) { - printf("."); - } else { - break; - } - } while (*s && *s != '-'); - } else { - char *e = strchr(s, '-'); - if (e == NULL) { - printf("%s", s); - } else { - printf("%.*s", (int)(e-s), s); - } - } - if (needdate) { - s = VersionTagDate; - char *e = strchr(s, '-'); - if (e == NULL) { - printf("-%s", s); - } else { - printf("-%.*s", (int)(e-s), s); - } - } -} - -} // namespace storage diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index ddfb0bd8a95..9445b35e222 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -7,7 +7,6 @@ #include <map> #include <sstream> #include <vespa/storageapi/message/persistence.h> -#include <vespa/storage/common/vtag.h> #include <vespa/fastlib/net/url.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/stringfmt.h> @@ -293,7 +292,7 @@ StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, } else { IndexPageReporter indexRep; indexRep << "<p><b>Binary version of Vespa:</b> " - << Vtag::currentVersion.toString() + << vespalib::Vtag::currentVersion.toString() << "</p>\n"; { std::vector<const framework::StatusReporter*> reporters( diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java index 6462ee1b2cd..cefa8d6e94a 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java @@ -52,9 +52,9 @@ class DocumentQueue { Document poll(long timeout, TimeUnit unit) throws InterruptedException { synchronized (queue) { - long startTime = System.currentTimeMillis(); long remainingToWait = unit.toMillis(timeout); while (queue.isEmpty()) { + long startTime = System.currentTimeMillis(); queue.wait(remainingToWait); remainingToWait -= (System.currentTimeMillis() - startTime); if (remainingToWait <= 0) { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java index 60324eda47a..facf29d245c 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java @@ -172,14 +172,6 @@ class IOThread implements Runnable, AutoCloseable { List<Document> getNextDocsForFeeding(int maxWaitUnits, TimeUnit timeUnit) { final List<Document> docsForSendChunk = new ArrayList<>(); - if (resultQueue.getPendingSize() > maxInFlightRequests) { - - // The queue is full do some sleep just to reduce network usage. - try { - stopSignal.await(300, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { /* Ignore */ } - return docsForSendChunk; - } int chunkSizeBytes = 0; try { drainFirstDocumentInQueueIfOld(); @@ -238,8 +230,16 @@ class IOThread implements Runnable, AutoCloseable { } } - // Return number of transient errors. - private int processResponse(InputStream serverResponse) throws IOException { + private static class ProcessResponse { + private final int transitiveErrorCount; + private final int processResultsCount; + ProcessResponse(int transitiveErrorCount, int processResultsCount) { + this.transitiveErrorCount = transitiveErrorCount; + this.processResultsCount = processResultsCount; + } + } + + private ProcessResponse processResponse(InputStream serverResponse) throws IOException { final Collection<EndpointResult> endpointResults = EndPointResultFactory.createResult(endpoint, serverResponse); statusReceivedCounter.addAndGet(endpointResults.size()); @@ -251,36 +251,47 @@ class IOThread implements Runnable, AutoCloseable { } resultQueue.resultReceived(endpointResult, clusterId); } - return transientErrors; + return new ProcessResponse(transientErrors, endpointResults.size()); } - // Returns number of transient errors. - private int feedDocumentAndProcessResults(List<Document> docs) + private ProcessResponse feedDocumentAndProcessResults(List<Document> docs) throws ServerResponseException, IOException { addDocumentsToResultQueue(docs); long startTime = System.currentTimeMillis(); InputStream serverResponse = sendAndReceive(docs); - int transientErrors = processResponse(serverResponse); + ProcessResponse processResponse = processResponse(serverResponse); lastGatewayProcessTimeMillis.set((int) (System.currentTimeMillis() - startTime)); - return transientErrors; + return processResponse; } - // Returns number of transient errors. - private int pullAndProcessData(int maxWaitTimeMilliSecs) + private ProcessResponse pullAndProcessData(int maxWaitTimeMilliSecs) throws ServerResponseException, IOException { - List<Document> nextDocsForFeeding = getNextDocsForFeeding(maxWaitTimeMilliSecs, TimeUnit.MILLISECONDS); - final int pendingResultQueueSize = resultQueue.getPendingSize(); pendingDocumentStatusCount.set(pendingResultQueueSize); + + List<Document> nextDocsForFeeding = (pendingResultQueueSize > maxInFlightRequests) + ? new ArrayList<>() // The queue is full, will not send more documents. + : getNextDocsForFeeding(maxWaitTimeMilliSecs, TimeUnit.MILLISECONDS); + + if (nextDocsForFeeding.isEmpty() && pendingResultQueueSize == 0) { //we have no unfinished business with the server now. log.finest("No document awaiting feeding, not waiting for results."); - return 0; + return new ProcessResponse(0, 0); } log.finest("Awaiting " + pendingResultQueueSize + " results."); - return feedDocumentAndProcessResults(nextDocsForFeeding); - + ProcessResponse processResponse = feedDocumentAndProcessResults(nextDocsForFeeding); + if (pendingResultQueueSize > maxInFlightRequests && processResponse.processResultsCount == 0) { + try { + // Max outstanding document operations, no more results on server side, wait a bit + // before asking again. + Thread.sleep(300); + } catch (InterruptedException e) { + // Ignore + } + } + return processResponse; } private ThreadState cycle(final ThreadState threadState) { @@ -319,8 +330,8 @@ class IOThread implements Runnable, AutoCloseable { case SESSION_SYNCED: final int maxWaitTimeMilliSecs = 100; try { - int transientErrors = pullAndProcessData(maxWaitTimeMilliSecs); - gatewayThrottler.handleCall(transientErrors); + ProcessResponse processResponse = pullAndProcessData(maxWaitTimeMilliSecs); + gatewayThrottler.handleCall(processResponse.transitiveErrorCount); } catch (ServerResponseException ser) { log.severe("Problems while handing data over to gateway " + endpoint + " " + ser.getMessage()); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java index 296a2c2015f..a91e170fbd5 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java @@ -73,8 +73,8 @@ public final class EndPointResultFactory { reply.errorCode.isTransient(), reply.traceMessage, exception)); - } catch (Exception e) { - throw new IllegalArgumentException("Bad result line from server: '" + line + "'", e); + } catch (Throwable t) { + throw new IllegalArgumentException("Bad result line from server: '" + line + "'", t); } } diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 422565f55ae..63bdc1eb5fa 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -1,5 +1,7 @@ <?xml version="1.0"?> <!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Implementation of document-api in the container. --> +<!-- TODO: Rename to container-documentapi on Vespa 7 --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> diff --git a/vespalib/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/vespalib/src/tests/eval/interpreted_function/interpreted_function_test.cpp index cffd5396bf4..9d01488dda1 100644 --- a/vespalib/src/tests/eval/interpreted_function/interpreted_function_test.cpp +++ b/vespalib/src/tests/eval/interpreted_function/interpreted_function_test.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/eval/function.h> +#include <vespa/vespalib/eval/tensor_spec.h> #include <vespa/vespalib/eval/interpreted_function.h> #include <vespa/vespalib/eval/test/eval_spec.h> #include <vespa/vespalib/eval/basic_nodes.h> @@ -113,4 +114,78 @@ TEST("require that basic addition works") { //----------------------------------------------------------------------------- +TEST("require that dot product like expression is not optimized for unknown types") { + const TensorEngine &engine = SimpleTensorEngine::ref(); + Function function = Function::parse("sum(a*b)"); + DoubleValue a(2.0); + DoubleValue b(3.0); + double expect = (2.0 * 3.0); + InterpretedFunction interpreted(engine, function, NodeTypes()); + EXPECT_EQUAL(4u, interpreted.program_size()); + InterpretedFunction::Context ctx; + ctx.add_param(a); + ctx.add_param(b); + const Value &result = interpreted.eval(ctx); + EXPECT_TRUE(result.is_double()); + EXPECT_EQUAL(expect, result.as_double()); +} + +TEST("require that dot product works with tensor function") { + const TensorEngine &engine = SimpleTensorEngine::ref(); + Function function = Function::parse("sum(a*b)"); + auto a = TensorSpec("tensor(x[3])") + .add({{"x", 0}}, 5.0) + .add({{"x", 1}}, 3.0) + .add({{"x", 2}}, 2.0); + auto b = TensorSpec("tensor(x[3])") + .add({{"x", 0}}, 7.0) + .add({{"x", 1}}, 11.0) + .add({{"x", 2}}, 13.0); + double expect = ((5.0 * 7.0) + (3.0 * 11.0) + (2.0 * 13.0)); + NodeTypes types(function, {ValueType::from_spec(a.type()), ValueType::from_spec(a.type())}); + InterpretedFunction interpreted(engine, function, types); + EXPECT_EQUAL(1u, interpreted.program_size()); + InterpretedFunction::Context ctx; + TensorValue va(engine.create(a)); + TensorValue vb(engine.create(b)); + ctx.add_param(va); + ctx.add_param(vb); + const Value &result = interpreted.eval(ctx); + EXPECT_TRUE(result.is_double()); + EXPECT_EQUAL(expect, result.as_double()); +} + +TEST("require that matrix multiplication works with tensor function") { + const TensorEngine &engine = SimpleTensorEngine::ref(); + Function function = Function::parse("sum(a*b,y)"); + auto a = TensorSpec("tensor(x[2],y[2])") + .add({{"x", 0},{"y", 0}}, 1.0) + .add({{"x", 0},{"y", 1}}, 2.0) + .add({{"x", 1},{"y", 0}}, 3.0) + .add({{"x", 1},{"y", 1}}, 5.0); + auto b = TensorSpec("tensor(y[2],z[2])") + .add({{"y", 0},{"z", 0}}, 7.0) + .add({{"y", 0},{"z", 1}}, 11.0) + .add({{"y", 1},{"z", 0}}, 13.0) + .add({{"y", 1},{"z", 1}}, 17.0); + auto expect = TensorSpec("tensor(x[2],z[2])") + .add({{"x", 0},{"z", 0}}, (1.0 * 7.0) + (2.0 * 13.0)) + .add({{"x", 0},{"z", 1}}, (1.0 * 11.0) + (2.0 * 17.0)) + .add({{"x", 1},{"z", 0}}, (3.0 * 7.0) + (5.0 * 13.0)) + .add({{"x", 1},{"z", 1}}, (3.0 * 11.0) + (5.0 * 17.0)); + NodeTypes types(function, {ValueType::from_spec(a.type()), ValueType::from_spec(a.type())}); + InterpretedFunction interpreted(engine, function, types); + EXPECT_EQUAL(1u, interpreted.program_size()); + InterpretedFunction::Context ctx; + TensorValue va(engine.create(a)); + TensorValue vb(engine.create(b)); + ctx.add_param(va); + ctx.add_param(vb); + const Value &result = interpreted.eval(ctx); + ASSERT_TRUE(result.is_tensor()); + EXPECT_EQUAL(expect, engine.to_spec(*result.as_tensor())); +} + +//----------------------------------------------------------------------------- + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp b/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp index 18d8a8dd508..f4edd8901e4 100644 --- a/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp +++ b/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp @@ -189,6 +189,52 @@ testTensorMapper(FixtureType &f) {{{"x","10"},{"y","1"}}, 7} }, { "x", "y" })); + TEST_DO(f.assertDenseMap({ + {{{"x",0},{"y",0}}, 1}, + {{{"x",0},{"y",1}}, 5}, + {{{"x",1},{"y",0}}, 3}, + {{{"x",1},{"y",1}}, 0} + }, + "tensor(x[2], y[])", + { + {{{"x","0"},{"y","0"}}, 1}, + {{{"x","1"},{"y","0"}}, 3}, + {{{"x","0"},{"y","1"}}, 5}, + {{{"x","10"},{"y","1"}}, 7} + }, + { "x", "y" })); + TEST_DO(f.assertDenseMap({ + {{{"x",0},{"y",0}}, 1}, + {{{"x",0},{"y",1}}, 5}, + {{{"x",1},{"y",0}}, 3}, + {{{"x",1},{"y",1}}, 0}, + {{{"x",2},{"y",0}}, 7}, + {{{"x",2},{"y",1}}, 0} + }, + "tensor(x[], y[])", + { + {{{"x","0"},{"y","0"}}, 1}, + {{{"x","1"},{"y","0"}}, 3}, + {{{"x","0"},{"y","1"}}, 5}, + {{{"x","2"},{"y","0"}}, 7} + }, + { "x", "y" })); + TEST_DO(f.assertDenseMap({ + {{{"x",0},{"y",0}}, 1}, + {{{"x",0},{"y",1}}, 5}, + {{{"x",0},{"y",2}}, 0}, + {{{"x",1},{"y",0}}, 3}, + {{{"x",1},{"y",1}}, 0}, + {{{"x",1},{"y",2}}, 0} + }, + "tensor(x[], y[3])", + { + {{{"x","0"},{"y","0"}}, 1}, + {{{"x","1"},{"y","0"}}, 3}, + {{{"x","0"},{"y","1"}}, 5}, + {{{"x","10"},{"y","3"}}, 7} + }, + { "x", "y" })); } TEST_F("test tensor mapper for SparseTensor", SparseFixture) diff --git a/vespalib/src/vespa/vespalib/component/CMakeLists.txt b/vespalib/src/vespa/vespalib/component/CMakeLists.txt index 36dba10f7d3..574cad33333 100644 --- a/vespalib/src/vespa/vespalib/component/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/component/CMakeLists.txt @@ -3,5 +3,7 @@ vespa_add_library(vespalib_vespalib_component OBJECT SOURCES version.cpp versionspecification.cpp + vtag.cpp DEPENDS ) +set_source_files_properties(vtag.cpp PROPERTIES COMPILE_FLAGS "${VTAG_DEFINES}") diff --git a/messagebus/src/vespa/messagebus/vtag.cpp b/vespalib/src/vespa/vespalib/component/vtag.cpp index 0a42b5e6ec0..c6619c29f1c 100644 --- a/messagebus/src/vespa/messagebus/vtag.cpp +++ b/vespalib/src/vespa/vespalib/component/vtag.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <string.h> #include <stdio.h> #include "vtag.h" -#include <vespa/vespalib/component/version.h> #ifndef V_TAG #define V_TAG "NOTAG" +#define V_TAG_TYPE "NOTAG" +#define V_TAG_VALUE "NOTAG" #define V_TAG_DATE "NOTAG" #define V_TAG_SYSTEM "NOTAG" #define V_TAG_SYSTEM_REV "NOTAG" @@ -15,7 +14,7 @@ #define V_TAG_ARCH "NOTAG" #endif -namespace mbus { +namespace vespalib { char VersionTag[] = V_TAG; char VersionTagDate[] = V_TAG_DATE; @@ -26,7 +25,7 @@ char VersionTagPkg[] = V_TAG_PKG; char VersionTagComponent[] = V_TAG_COMPONENT; char VersionTagArch[] = V_TAG_ARCH; -vespalib::Version Vtag::currentVersion(VersionTagComponent); +Version Vtag::currentVersion(VersionTagComponent); void Vtag::printVersionNice() @@ -78,4 +77,4 @@ Vtag::printVersionNice() } } -} // namespace mbus +} diff --git a/storage/src/vespa/storage/common/vtag.h b/vespalib/src/vespa/vespalib/component/vtag.h index 26236f13a41..b8f07063a51 100644 --- a/storage/src/vespa/storage/common/vtag.h +++ b/vespalib/src/vespa/vespalib/component/vtag.h @@ -1,24 +1,26 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #pragma once -namespace vespalib { - class Version; -} +#include "version.h" -namespace storage { +namespace vespalib { extern char VersionTag[]; +extern char VersionTagType[]; +extern char VersionTagValue[]; extern char VersionTagDate[]; extern char VersionTagSystem[]; extern char VersionTagSystemRev[]; extern char VersionTagBuilder[]; +extern char VersionTagPkg[]; +extern char VersionTagComponent[]; +extern char VersionTagArch[]; + class Vtag { public: - static vespalib::Version currentVersion; + static Version currentVersion; static void printVersionNice(); }; -} // namespace messagebus - +} diff --git a/vespalib/src/vespa/vespalib/eval/interpreted_function.cpp b/vespalib/src/vespa/vespalib/eval/interpreted_function.cpp index 049d2ef81c4..bb528aedb85 100644 --- a/vespalib/src/vespa/vespalib/eval/interpreted_function.cpp +++ b/vespalib/src/vespa/vespalib/eval/interpreted_function.cpp @@ -107,8 +107,6 @@ void op_tensor_sum(State &state, uint64_t) { const eval::Tensor *tensor = state.peek(0).as_tensor(); if (tensor != nullptr) { state.replace(1, tensor->engine().reduce(*tensor, operation::Add(), {}, state.stash)); - } else { - state.replace(1, state.stash.create<ErrorValue>()); } } @@ -124,6 +122,47 @@ void op_tensor_sum_dimension(State &state, uint64_t param) { //----------------------------------------------------------------------------- +template <typename T> +const T &undef_cref() { + const T *undef = nullptr; + assert(undef); + return *undef; +} + +struct TensorFunctionArgArgMeta { + TensorFunction::UP function; + size_t param1; + size_t param2; + TensorFunctionArgArgMeta(TensorFunction::UP function_in, size_t param1_in, size_t param2_in) + : function(std::move(function_in)), param1(param1_in), param2(param2_in) {} +}; + +struct ArgArgInput : TensorFunction::Input { + const TensorFunctionArgArgMeta &meta; + const State &state; + ArgArgInput(const TensorFunctionArgArgMeta &meta_in, const State &state_in) + : meta(meta_in), state(state_in) {} + const Value &get_tensor(size_t id) const override { + if (id == 0) { + return state.params[meta.param1]; + } else if (id == 1) { + return state.params[meta.param2]; + } + return undef_cref<Value>(); + } + const UnaryOperation &get_map_operation(size_t) const override { + return undef_cref<UnaryOperation>(); + } +}; + +void op_tensor_function_arg_arg(State &state, uint64_t param) { + const TensorFunctionArgArgMeta &meta = unwrap_param<TensorFunctionArgArgMeta>(param); + ArgArgInput input(meta, state); + state.stack.push_back(meta.function->eval(input, state.stash)); +} + +//----------------------------------------------------------------------------- + struct ProgramBuilder : public NodeVisitor, public NodeTraverser { std::vector<Instruction> &program; Stash &stash; @@ -135,6 +174,29 @@ struct ProgramBuilder : public NodeVisitor, public NodeTraverser { //------------------------------------------------------------------------- + bool is_typed_tensor(const Node &node) const { + const ValueType &type = types.get_type(node); + return (type.is_tensor() && !type.dimensions().empty()); + } + + bool is_typed(const Node &node) const { + return (types.get_type(node).is_double() || is_typed_tensor(node)); + } + + bool is_typed_tensor_param(const Node &node) const { + auto sym = as<Symbol>(node); + return (sym && (sym->id() >= 0) && is_typed_tensor(node)); + } + + bool is_typed_tensor_product_of_params(const Node &node) const { + auto mul = as<Mul>(node); + return (mul && is_typed_tensor(*mul) && + is_typed_tensor_param(mul->lhs()) && + is_typed_tensor_param(mul->rhs())); + } + + //------------------------------------------------------------------------- + virtual void visit(const Number &node) { program.emplace_back(op_load_const, wrap_param<Value>(stash.create<DoubleValue>(node.value()))); } @@ -206,7 +268,24 @@ struct ProgramBuilder : public NodeVisitor, public NodeTraverser { wrap_param<Value>(stash.create<TensorValue>(std::move(tensor)))); } virtual void visit(const TensorSum &node) { - if (node.dimension().empty()) { + if (is_typed(node) && is_typed_tensor_product_of_params(node.get_child(0))) { + assert(program.size() >= 3); // load,load,mul + program.pop_back(); // mul + program.pop_back(); // load + program.pop_back(); // load + std::vector<vespalib::string> dim_list; + if (!node.dimension().empty()) { + dim_list.push_back(node.dimension()); + } + auto a = as<Symbol>(node.get_child(0).get_child(0)); + auto b = as<Symbol>(node.get_child(0).get_child(1)); + auto ir = tensor_function::reduce(tensor_function::apply(operation::Mul(), + tensor_function::inject(types.get_type(*a), 0), + tensor_function::inject(types.get_type(*b), 1)), operation::Add(), dim_list); + auto fun = tensor_engine.compile(std::move(ir)); + const auto &meta = stash.create<TensorFunctionArgArgMeta>(std::move(fun), a->id(), b->id()); + program.emplace_back(op_tensor_function_arg_arg, wrap_param<TensorFunctionArgArgMeta>(meta)); + } else if (node.dimension().empty()) { program.emplace_back(op_tensor_sum); } else { program.emplace_back(op_tensor_sum_dimension, diff --git a/vespalib/src/vespa/vespalib/eval/interpreted_function.h b/vespalib/src/vespa/vespalib/eval/interpreted_function.h index 73d0f237131..615f9d4eb82 100644 --- a/vespalib/src/vespa/vespalib/eval/interpreted_function.h +++ b/vespalib/src/vespa/vespalib/eval/interpreted_function.h @@ -91,6 +91,7 @@ public: InterpretedFunction(const TensorEngine &engine, const Function &function, const NodeTypes &types) : InterpretedFunction(engine, function.root(), function.num_params(), types) {} InterpretedFunction(InterpretedFunction &&rhs) = default; + size_t program_size() const { return _program.size(); } size_t num_params() const { return _num_params; } const Value &eval(Context &ctx) const; }; diff --git a/vespalib/src/vespa/vespalib/eval/test/eval_spec.cpp b/vespalib/src/vespa/vespalib/eval/test/eval_spec.cpp index dda63546506..9e6dfd4dbc5 100644 --- a/vespalib/src/vespa/vespalib/eval/test/eval_spec.cpp +++ b/vespalib/src/vespa/vespalib/eval/test/eval_spec.cpp @@ -112,12 +112,14 @@ EvalSpec::add_function_call_cases() { .add_case({my_nan}, 1.0).add_case({my_inf}, 0.0).add_case({-my_inf}, 0.0); add_rule({"a", -1.0, 1.0}, "relu(a)", [](double a){ return std::max(a, 0.0); }); add_rule({"a", -1.0, 1.0}, "sigmoid(a)", [](double a){ return 1.0 / (1.0 + std::exp(-1.0 * a)); }); + add_rule({"a", -1.0, 1.0}, "sum(a)", [](double a){ return a; }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "atan2(a,b)", [](double a, double b){ return std::atan2(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "ldexp(a,b)", [](double a, double b){ return std::ldexp(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "pow(a,b)", [](double a, double b){ return std::pow(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "fmod(a,b)", [](double a, double b){ return std::fmod(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "min(a,b)", [](double a, double b){ return std::min(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "max(a,b)", [](double a, double b){ return std::max(a, b); }); + add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "match(a,b)", [](double a, double b){ return (a * b); }); } void diff --git a/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp b/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp index 7b218d80a85..d8d27482f49 100644 --- a/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp +++ b/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp @@ -59,7 +59,7 @@ DefaultTensorEngine::to_spec(const Tensor &tensor) const eval::TensorFunction::UP DefaultTensorEngine::compile(eval::tensor_function::Node_UP expr) { - return DenseTensorFunctionCompiler::compile(std::move(expr)); + return std::move(expr); } struct IsAddOperation : public eval::DefaultOperationVisitor { diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_view.h b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_view.h index f26174a4ff9..b6d96469ed4 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_view.h +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_view.h @@ -25,6 +25,7 @@ public: private: const eval::ValueType &_type; +protected: CellsRef _cells; public: diff --git a/vespalib/src/vespa/vespalib/tensor/dense/mutable_dense_tensor_view.h b/vespalib/src/vespa/vespalib/tensor/dense/mutable_dense_tensor_view.h new file mode 100644 index 00000000000..ad6468dc8d4 --- /dev/null +++ b/vespalib/src/vespa/vespalib/tensor/dense/mutable_dense_tensor_view.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "dense_tensor_view.h" + +namespace vespalib { +namespace tensor { + +/** + * A mutable view to a dense tensor where all dimensions are indexed. + */ +class MutableDenseTensorView : public DenseTensorView +{ + eval::ValueType _concreteType; + +public: + MutableDenseTensorView(eval::ValueType type_in, CellsRef cells_in) + : DenseTensorView(_concreteType, cells_in), + _concreteType(type_in) + { + } + + CellsRef &cells() { return _cells; } + eval::ValueType &type() { return _concreteType; } +}; + +} // namespace vespalib::tensor +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp index f740ffbf348..9c263fb3ffa 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp @@ -98,15 +98,124 @@ SparseTensorMapper<TensorT>::map(const Tensor &tensor, return mapper.build(); } +static constexpr uint32_t BAD_LABEL = std::numeric_limits<uint32_t>::max(); +static constexpr uint32_t BAD_ADDRESS = std::numeric_limits<uint32_t>::max(); + +uint32_t mapLabelToNumber(vespalib::stringref label) { + uint32_t result = 0; + for (char c : label) { + if (c < '0' || c > '9') { + return BAD_LABEL; // bad char + } + result = result * 10 + (c - '0'); + if (result > 100000000) { + return BAD_LABEL; // overflow + } + } + return result; +} + +class DenseTensorTypeMapper : public TensorVisitor +{ + ValueType _type; + std::vector<ValueType::Dimension> _dimensions; + + bool addressOK(const TensorAddress &address); + void expandUnboundDimensions(const TensorAddress &address); + + virtual void visit(const TensorAddress &address, double value) override; + + DenseTensorTypeMapper(const ValueType &type); + ~DenseTensorTypeMapper(); + + ValueType build(); +public: + static ValueType map(const Tensor &tensor, const ValueType &type); +}; + +bool +DenseTensorTypeMapper::addressOK(const TensorAddress &address) +{ + TensorAddressElementIterator<TensorAddress> addressIterator(address); + auto dimIterator = _dimensions.begin(); + for (const auto &dimension : _type.dimensions()) { + if (addressIterator.skipToDimension(dimension.name)) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if (label == BAD_LABEL || + (dimension.is_bound() && label >= dimIterator->size)) { + return false; + } + addressIterator.next(); + } + ++dimIterator; + } + assert(dimIterator == _dimensions.end()); + return true; +} + + +void +DenseTensorTypeMapper::expandUnboundDimensions(const TensorAddress &address) +{ + TensorAddressElementIterator<TensorAddress> addressIterator(address); + auto dimIterator = _dimensions.begin(); + for (const auto &dimension : _type.dimensions()) { + if (addressIterator.skipToDimension(dimension.name)) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if (label != BAD_LABEL && + !dimension.is_bound() && + label >= dimIterator->size) { + dimIterator->size = label + 1; + } + addressIterator.next(); + } + ++dimIterator; + } + assert(dimIterator == _dimensions.end()); +} + +void +DenseTensorTypeMapper::visit(const TensorAddress &address, double value) +{ + (void) value; + if (addressOK(address)) { + expandUnboundDimensions(address); + } +} + +DenseTensorTypeMapper::DenseTensorTypeMapper(const ValueType &type) + : _type(type), + _dimensions(type.dimensions()) +{ + for (auto &dimension : _dimensions) { + if (!dimension.is_bound()) + dimension.size = 1; + } +} + +DenseTensorTypeMapper::~DenseTensorTypeMapper() +{ +} + +ValueType +DenseTensorTypeMapper::build() +{ + return ValueType::tensor_type(std::move(_dimensions)); +} + +ValueType +DenseTensorTypeMapper::map(const Tensor &tensor, const ValueType &type) +{ + DenseTensorTypeMapper mapper(type); + tensor.accept(mapper); + return mapper.build(); +} + class DenseTensorMapper : public TensorVisitor { eval::ValueType _type; DenseTensor::Cells _cells; - static constexpr uint32_t BAD_LABEL = std::numeric_limits<uint32_t>::max(); - static constexpr uint32_t BAD_ADDRESS = - std::numeric_limits<uint32_t>::max(); - static uint32_t mapLabelToNumber(vespalib::stringref label); uint32_t mapAddressToIndex(const TensorAddress &address); virtual void visit(const TensorAddress &address, double value) override; @@ -142,22 +251,6 @@ DenseTensorMapper::build() } uint32_t -DenseTensorMapper::mapLabelToNumber(vespalib::stringref label) -{ - uint32_t result = 0; - for (char c : label) { - if (c < '0' || c > '9') { - return BAD_LABEL; // bad char - } - result = result * 10 + (c - '0'); - if (result > 100000000) { - return BAD_LABEL; // overflow - } - } - return result; -} - -uint32_t DenseTensorMapper::mapAddressToIndex(const TensorAddress &address) { uint32_t idx = 0; @@ -191,7 +284,9 @@ DenseTensorMapper::visit(const TensorAddress &address, double value) std::unique_ptr<Tensor> DenseTensorMapper::map(const Tensor &tensor, const ValueType &type) { - DenseTensorMapper mapper(type); + DenseTensorMapper mapper(type.is_abstract() ? + DenseTensorTypeMapper::map(tensor, type) : + type); tensor.accept(mapper); return mapper.build(); } |