diff options
44 files changed, 360 insertions, 618 deletions
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 63313ecabe1..9a975f1b727 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -1145,12 +1145,14 @@ ], "methods" : [ "public static com.yahoo.config.model.api.EndpointCertificateMetadata$Provider[] values()", - "public static com.yahoo.config.model.api.EndpointCertificateMetadata$Provider valueOf(java.lang.String)" + "public static com.yahoo.config.model.api.EndpointCertificateMetadata$Provider valueOf(java.lang.String)", + "public boolean acme()" ], "fields" : [ "public static final enum com.yahoo.config.model.api.EndpointCertificateMetadata$Provider digicert", "public static final enum com.yahoo.config.model.api.EndpointCertificateMetadata$Provider globalsign", - "public static final enum com.yahoo.config.model.api.EndpointCertificateMetadata$Provider zerossl" + "public static final enum com.yahoo.config.model.api.EndpointCertificateMetadata$Provider zerossl", + "public static final enum com.yahoo.config.model.api.EndpointCertificateMetadata$Provider letsencrypt" ] }, "com.yahoo.config.model.api.EndpointCertificateMetadata" : { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java index 591677f6677..eaed57e9154 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java @@ -5,7 +5,23 @@ import java.util.Objects; public class EndpointCertificateMetadata { - public enum Provider { digicert, globalsign, zerossl } + public enum Provider { + + digicert(false), + globalsign(false), + zerossl(true), + letsencrypt(true); + + private final boolean acme; + + Provider(boolean acme) { + this.acme = acme; + } + + public boolean acme() { return acme; } + + } + private final String keyName; private final String certName; private final int version; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index e76e40f2235..e8103f1d1df 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -55,7 +55,6 @@ import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.vespa.model.ml.ModelName; import com.yahoo.vespa.model.ml.OnnxModelInfo; import com.yahoo.vespa.model.routing.Routing; -import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import org.xml.sax.SAXException; @@ -207,9 +206,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Mode .map(type -> type.getFullName().getName()) .collect(Collectors.toCollection(LinkedHashSet::new)); - Set<String> typesWithIndexedFields = content.getSearch().getIndexed() == null + Set<String> typesWithIndexedFields = content.getSearch().getSearchCluster() == null ? Set.of() - : content.getSearch().getIndexed().schemas().values().stream() + : content.getSearch().getSearchCluster().schemas().values().stream() .filter(schemaInfo -> schemaInfo.fullSchema() .allConcreteFields() .stream().anyMatch(SDField::doesIndexing)) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java index 5117fb7272c..f95ae0b6153 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -8,10 +8,10 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.application.validation.change.search.DocumentDatabaseChangeValidator; -import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; import java.util.ArrayList; import java.util.List; @@ -29,7 +29,7 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { public void validate(ChangeContext context) { for (Map.Entry<String, ContentCluster> currentEntry : context.previousModel().getContentClusters().entrySet()) { ContentCluster nextCluster = context.model().getContentClusters().get(currentEntry.getKey()); - if (nextCluster != null && nextCluster.getSearch().hasIndexedCluster()) { + if (nextCluster != null && nextCluster.getSearch().hasSearchCluster()) { validateContentCluster(currentEntry.getValue(), nextCluster, context.deployState()).forEach(context::require); } } @@ -47,9 +47,9 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { DeployState deployState) { List<ConfigChangeAction> result = new ArrayList<>(); - for (DocumentDatabase currentDb : getDocumentDbs(currentCluster.getSearch())) { + for (DocumentDatabase currentDb : getDocumentDbs(currentCluster.getSearch().getSearchCluster())) { String docTypeName = currentDb.getName(); - var nextDb = nextCluster.getSearch().getIndexed().getDocumentDB(docTypeName); + var nextDb = nextCluster.getSearch().getSearchCluster().getDocumentDB(docTypeName); if (nextDb != null) { result.addAll(validateDocumentDatabase(currentCluster, nextCluster, docTypeName, currentDb, nextDb, deployState)); @@ -71,14 +71,14 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { new DocumentDatabaseChangeValidator(currentCluster.id(), currentDb, currentDocType, nextDb, nextDocType, deployState).validate(); - return modifyActions(result, getSearchNodeServices(nextCluster.getSearch().getIndexed()), docTypeName); + return modifyActions(result, getSearchNodeServices(nextCluster.getSearch().getSearchCluster()), docTypeName); } - private static List<DocumentDatabase> getDocumentDbs(ContentSearchCluster cluster) { - if (cluster.getIndexed() != null) { - return cluster.getIndexed().getDocumentDbs(); + private static List<DocumentDatabase> getDocumentDbs(SearchCluster cluster) { + if (cluster != null) { + return cluster.getDocumentDbs(); } - return new ArrayList<>(); + return List.of(); } private static List<ServiceInfo> getSearchNodeServices(IndexedSearchCluster cluster) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 0d42219dade..7c1e0def42d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -30,13 +30,11 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> { ContentCluster nextCluster = context.model().getContentClusters().get(clusterName); if (nextCluster != null) { - var nextStreamingClusters = nextCluster.getSearch().getClusters(); - currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> { - SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName()); - if (nextStreamingCluster != null) { - validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require); - } - }); + if (currentCluster.getSearch().getSearchCluster() != null && nextCluster.getSearch().getSearchCluster() != null) { + validateStreamingCluster(currentCluster, currentCluster.getSearch().getSearchCluster(), + nextCluster, nextCluster.getSearch().getSearchCluster()) + .forEach(context::require); + } } }); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index de9e9ad7a4b..a71d1025ece 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; -import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.dispatch.Dispatcher; @@ -169,25 +168,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> implements @Override public void getConfig(QrSearchersConfig.Builder builder) { - for (int i = 0; i < searchClusters.size(); i++) { - SearchCluster sys = findClusterWithId(searchClusters, i); - var scB = new QrSearchersConfig.Searchcluster.Builder().name(sys.getClusterName()); - for (SchemaInfo spec : sys.schemas().values()) { - scB.searchdef(spec.fullSchema().getName()); - } - scB.rankprofiles_configid(sys.getConfigId()); - scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); - scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(sys.getStorageRouteSpec())); - builder.searchcluster(scB); - } - } - - private static SearchCluster findClusterWithId(List<SearchCluster> clusters, int index) { - for (SearchCluster sys : clusters) { - if (sys.getClusterIndex() == index) - return sys; - } - throw new IllegalArgumentException("No search cluster with index " + index + " exists"); + searchClusters.forEach(sc -> builder.searchcluster(sc.getQrSearcherConfig())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index da7dfdc7b84..554dacb39e3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.model.search.SearchCluster; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; -import java.util.Objects; import java.util.Set; /** @@ -38,12 +37,7 @@ public class LocalProvider extends Provider implements @Override public void getConfig(ClusterConfig.Builder builder) { - Objects.requireNonNull(searchCluster, "Null search cluster!"); - builder.clusterId(searchCluster.getClusterIndex()); - builder.clusterName(searchCluster.getClusterName()); - - if (searchCluster.getVisibilityDelay() != null) - builder.cacheTimeout(convertVisibilityDelay(searchCluster.getVisibilityDelay())); + searchCluster.getConfig(builder); } @Override @@ -131,13 +125,4 @@ public class LocalProvider extends Provider implements searchCluster.getConfig(builder); } - // The semantics of visibility delay in search is deactivating caches if the - // delay is less than 1.0, in qrs the cache is deactivated if the delay is 0 - // (or less). 1.0 seems a little arbitrary, so just doing the conversion - // here instead of having two totally independent implementations having to - // follow each other down in the modules. - private static Double convertVisibilityDelay(Double visibilityDelay) { - return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay; - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java index 3f1de25f71f..3cf887a5928 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java @@ -41,9 +41,17 @@ public class SearchChains extends Chains<SearchChain> { private void setSearchClusterForLocalProvider(Map<String, ? extends SearchCluster> clusterIndexByName) { for (LocalProvider provider : localProviders()) { - SearchCluster cluster = clusterIndexByName.get(provider.getClusterName()); - if (cluster == null) - throw new IllegalArgumentException("No searchable content cluster with id '" + provider.getClusterName() + "'"); + String clusterName = provider.getClusterName(); + SearchCluster cluster = clusterIndexByName.get(clusterName); + if (cluster == null) { + if (clusterName.contains(".")) { // Is there a super cluster ... + String prefix = clusterName.substring(0, clusterName.indexOf('.')); + cluster = clusterIndexByName.get(prefix); + } + if (cluster == null) { + throw new IllegalArgumentException("No searchable content cluster with id '" + provider.getClusterName() + "'"); + } + } provider.setSearchCluster(cluster); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 1254f8e110a..3ccc44fe60b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -135,8 +135,12 @@ public class Content extends ConfigModel { public static List<SearchCluster> getSearchClusters(ConfigModelRepo pc) { List<SearchCluster> clusters = new ArrayList<>(); - for (ContentCluster c : getContentClusters(pc)) - clusters.addAll(c.getSearch().getClusters().values()); + for (ContentCluster c : getContentClusters(pc)) { + SearchCluster sc = c.getSearch().getSearchCluster(); + if (sc != null) { + clusters.add(sc); + } + } return clusters; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index a92362bf899..ebb556a2ba2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -31,9 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.LinkedHashMap; -import java.util.Optional; import java.util.Objects; -import java.util.TreeMap; import java.util.function.Predicate; /** @@ -53,11 +50,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> private final boolean flushOnShutdown; private final Boolean syncTransactionLog; - /** If this is set up for streaming search, it is modelled as one search cluster per search definition */ - private final Map<String, SearchCluster> clusters = new TreeMap<>(); - /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ - private IndexedSearchCluster indexedCluster; + private IndexedSearchCluster searchCluster; private final IndexingDocproc indexingDocproc; private Redundancy redundancy; @@ -115,8 +109,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } search.setResourceLimits(resourceLimits); - buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); - buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); + buildSearchCluster(deployState, clusterElem, clusterName, search); return search; } @@ -128,73 +121,18 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> return clusterElem.childAsDouble("engine.proton.query-timeout"); } - private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) { - var schemas = deployState.getSchemas(); - for (var candidate : schemas) { - if (candidate.getName().equals(docTypeName)) { - return candidate; - } - } - return null; - } - - private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { + private void buildSearchCluster(DeployState deployState, ModelElement clusterElem, + String clusterName, ContentSearchCluster search) { ModelElement docElem = clusterElem.child("documents"); + if (docElem == null) return; - if (docElem == null) { - return; - } - - for (ModelElement docType : docElem.subElements("document")) { - String docTypeName = docType.stringAttribute("type"); - String mode = docType.stringAttribute("mode"); - var schema = findResponsibleSchema(deployState, docTypeName); - if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) { - buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); - } - } - } - - private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, - ContentSearchCluster search, ModelElement docType) { - String docTypeName = docType.stringAttribute("type"); - StreamingSearchCluster cluster = new StreamingSearchCluster(search, - clusterName + "." + docTypeName, - 0, - docTypeName, - clusterName); - search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType)); - } - - private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, - String clusterName, ContentSearchCluster search) { - List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); - if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); - - Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); - if (visibilityDelay != null) { - search.setVisibilityDelay(visibilityDelay); - } - - search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); - } - } - - private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { - List<ModelElement> indexedDefs = new ArrayList<>(); - ModelElement docElem = clusterElem.child("documents"); - if (docElem == null) { - return indexedDefs; + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + search.setVisibilityDelay(visibilityDelay); } - for (ModelElement docType : docElem.subElements("document")) { - String mode = docType.stringAttribute("mode"); - if ("index".equals(mode)) { - indexedDefs.add(docType); - } - } - return indexedDefs; + var isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); + search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document")); } } @@ -233,21 +171,24 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public void setVisibilityDelay(double delay) { this.visibilityDelay=delay; - if (hasIndexedCluster()) { - indexedCluster.setVisibilityDelay(delay); + if (searchCluster != null) { + searchCluster.setVisibilityDelay(delay); } } - private void addSearchCluster(DeployState deployState, SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { + private void addSearchCluster(DeployState deployState, IndexedSearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { addSchemas(deployState, documentDefs, cluster); if (queryTimeout != null) { cluster.setQueryTimeout(queryTimeout); } cluster.deriveFromSchemas(deployState); - addCluster(cluster); + if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) { + addCluster(cluster); + } } + private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); @@ -255,22 +196,17 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (schema == null) throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); + if (schema.isDocumentsOnly()) continue; sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null)); } } - private void addCluster(SearchCluster sc) { - if (clusters.containsKey(sc.getClusterName())) { - throw new IllegalArgumentException("Duplicate cluster '" + sc.getClusterName() + "'"); + private void addCluster(IndexedSearchCluster sc) { + if (searchCluster != null) { + throw new IllegalArgumentException("Duplicate indexed cluster '" + searchCluster.getClusterName() + "'"); } - if (sc instanceof IndexedSearchCluster) { - if (indexedCluster != null) { - throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); - } - indexedCluster = (IndexedSearchCluster)sc; - } - clusters.put(sc.getClusterName(), sc); + searchCluster = sc; } /** @@ -280,24 +216,26 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> * with indexing, null if it has both or none. */ public Boolean isStreaming() { - boolean hasStreaming = false; - boolean hasIndexed = false; - for (var cluster : clusters.values()) { - if (cluster.hasStreaming()) - hasStreaming = true; - else - hasIndexed = true; - } - if (hasIndexed == hasStreaming) return null; + if (searchCluster == null) return false; + boolean hasStreaming = searchCluster.hasStreaming(); + if (searchCluster.hasIndexed() == hasStreaming) return null; return hasStreaming; } + public boolean hasStreaming() { + return (searchCluster != null) && searchCluster.hasStreaming(); + } + + public boolean hasIndexed() { + return (searchCluster != null) && searchCluster.hasIndexed(); + } + public List<SearchNode> getSearchNodes() { - return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed; + return (searchCluster != null) ? searchCluster.getSearchNodes() : nonIndexed; } public void addSearchNode(DeployState deployState, ContentNode node, StorageGroup parentGroup, ModelElement element) { - TreeConfigProducer<AnyConfigProducer> parent = hasIndexedCluster() ? getIndexed() : this; + TreeConfigProducer<AnyConfigProducer> parent = (searchCluster != null) ? searchCluster : this; NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; @@ -319,8 +257,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } searchNode.setTls(tls); - if (hasIndexedCluster()) { - getIndexed().addSearcher(searchNode); + if (searchCluster != null) { + searchCluster.addSearcher(searchNode); } else { nonIndexed.add(searchNode); } @@ -347,28 +285,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } public boolean usesHierarchicDistribution() { - return indexedCluster != null && groupToSpecMap.size() > 1; + return searchCluster != null && groupToSpecMap.size() > 1; } public void handleRedundancy(Redundancy redundancy) { this.redundancy = redundancy; } - private Optional<StreamingSearchCluster> findStreamingCluster(String docType) { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .filter(ssc -> ssc.schemas().get(docType) != null) - .findFirst(); - } - - public List<StreamingSearchCluster> getStreamingClusters() { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .toList(); - } - public List<NewDocumentType> getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); } public List<NewDocumentType> getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); } public List<NewDocumentType> getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); } @@ -380,13 +303,15 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } private boolean hasIndexingModeStreaming(NewDocumentType type) { - return findStreamingCluster(type.getFullName().getName()).isPresent(); + if (searchCluster == null) return false; + var schemaInfo = searchCluster.schemas().get(type.getName()); + return (schemaInfo != null) && (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STREAMING); } private boolean hasIndexingModeIndexed(NewDocumentType type) { - return !hasIndexingModeStreaming(type) - && hasIndexedCluster() - && getIndexed().hasDocumentDB(type.getFullName().getName()); + if (searchCluster == null) return false; + var schemaInfo = searchCluster.schemas().get(type.getName()); + return (schemaInfo != null) && (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.INDEX); } private boolean hasIndexingModeStoreOnly(NewDocumentType type) { @@ -395,7 +320,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(ProtonConfig.Builder builder) { - boolean hasAnyNonIndexedCluster = false; + boolean hasAnyNonIndexedSchema = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -407,13 +332,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { - hasAnyNonIndexedCluster = true; - findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + hasAnyNonIndexedSchema = true; + searchCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { - getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + searchCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); } else { - hasAnyNonIndexedCluster = true; + hasAnyNonIndexedSchema = true; ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { @@ -422,7 +347,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> builder.documentdb(ddbB); } - if (hasAnyNonIndexedCluster) { + if (hasAnyNonIndexedSchema) { builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2)); } else { builder.feeding.concurrency(defaultFeedConcurrency); @@ -465,20 +390,19 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(DispatchNodesConfig.Builder builder) { - if (hasIndexedCluster()) { - getIndexed().getConfig(builder); + if (searchCluster != null) { + searchCluster.getConfig(builder); } } @Override public void getConfig(DispatchConfig.Builder builder) { - if (hasIndexedCluster()) { - getIndexed().getConfig(builder); + if (searchCluster != null) { + searchCluster.getConfig(builder); } } - public Map<String, SearchCluster> getClusters() { return clusters; } - public IndexedSearchCluster getIndexed() { return indexedCluster; } - public boolean hasIndexedCluster() { return indexedCluster != null; } + public IndexedSearchCluster getSearchCluster() { return searchCluster; } + public boolean hasSearchCluster() { return searchCluster != null; } public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } public String getClusterName() { return clusterName; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 791faa401ed..bac86e37e8f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -73,14 +73,14 @@ import static java.util.logging.Level.WARNING; * @author bratseth */ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implements - DistributionConfig.Producer, - StorDistributionConfig.Producer, - StorDistributormanagerConfig.Producer, - FleetcontrollerConfig.Producer, - MetricsmanagerConfig.Producer, - MessagetyperouteselectorpolicyConfig.Producer, - BucketspacesConfig.Producer { - + DistributionConfig.Producer, + StorDistributionConfig.Producer, + StorDistributormanagerConfig.Producer, + FleetcontrollerConfig.Producer, + MetricsmanagerConfig.Producer, + MessagetyperouteselectorpolicyConfig.Producer, + BucketspacesConfig.Producer +{ private final String documentSelection; private ContentSearchCluster search; private final boolean isHosted; @@ -140,7 +140,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem c.search.handleRedundancy(c.redundancy); setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); - if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) + if (c.search.hasIndexed() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) throw new IllegalArgumentException("Indexed search requires proton as engine"); if (documentsElement != null) { @@ -177,8 +177,9 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (visibilityDelay != null) { csc.setVisibilityDelay(visibilityDelay); } - if (csc.hasIndexedCluster()) { - setupIndexedCluster(csc.getIndexed(), search, element, logger); + IndexedSearchCluster sc = csc.getSearchCluster(); + if (sc != null) { + setupIndexedCluster(sc, search, element, logger); } } @@ -530,7 +531,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem super.validate(); if (search.usesHierarchicDistribution() && !isHosted) { // validate manually configured groups - new IndexedHierarchicDistributionValidator(rootGroup, redundancy, search.getIndexed().getTuning().dispatch.getDispatchPolicy()).validate(); + new IndexedHierarchicDistributionValidator(rootGroup, redundancy, search.getSearchCluster().getTuning().dispatch.getDispatchPolicy()).validate(); } new ReservedDocumentTypeNameValidator().validate(documentDefinitions); new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java index 7fc713f81ef..f4e6244fa25 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -13,7 +13,7 @@ public class EngineFactoryBuilder { public PersistenceEngine.PersistenceFactory build(ModelElement clusterElem, ContentCluster c) { ModelElement persistence = clusterElem.child("engine"); if (persistence != null) { - if (c.getSearch().hasIndexedCluster() && persistence.child("proton") == null) { + if (c.getSearch().hasIndexed() && persistence.child("proton") == null) { throw new IllegalArgumentException("Persistence engine does not allow for indexed search. Please use <proton> as your engine."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4a37b27d1c7..c9c7e424ee6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -2,12 +2,8 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.schema.DocumentOnlySchema; -import com.yahoo.schema.derived.DerivedConfiguration; -import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; import com.yahoo.vespa.config.search.DispatchNodesConfig; @@ -45,9 +41,6 @@ public class IndexedSearchCluster extends SearchCluster implements summaryDecodePolicy = featureFlags.summaryDecodePolicy(); } - @Override - protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } - public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); } @@ -60,16 +53,6 @@ public class IndexedSearchCluster extends SearchCluster implements } public Tuning getTuning() { return tuning; } - @Override - public void deriveFromSchemas(DeployState deployState) { - for (SchemaInfo spec : schemas().values()) { - if (spec.fullSchema() instanceof DocumentOnlySchema) continue; - var db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode())); - add(db); - } - } - public void setSearchCoverage(SearchCoverage searchCoverage) { this.searchCoverage = searchCoverage; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index fc4b96ec384..8a60930664e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -3,18 +3,33 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -36,6 +51,50 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer private Double visibilityDelay = 0.0; private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>(); private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); + private final Map<String, AttributesProducer> documentDBProducerForStreaming = new HashMap<>(); + private final List<LegacyStreamingProxy> legacyproxy = new ArrayList<>(); + + private static class LegacyStreamingProxy extends TreeConfigProducer<AnyConfigProducer> implements + AttributesConfig.Producer, + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer, + JuniperrcConfig.Producer, + SummaryConfig.Producer, + VsmsummaryConfig.Producer, + VsmfieldsConfig.Producer + { + private final DocumentDatabase db; + LegacyStreamingProxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, + String schemaName, DerivedConfiguration derived) { + super(parent, "cluster." + clusterName + "." + schemaName); + this.db = new DocumentDatabase(this, schemaName, derived); + } + @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); } + } + + private static class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { + private final DerivedConfiguration derived; + + AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType, DerivedConfiguration derived) { + super(parent, docType); + this.derived = derived; + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + derived.getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } + } public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -48,9 +107,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public void add(SchemaInfo schema) { schemas.put(schema.name(), schema); } - public void add(DocumentDatabase db) { - documentDbs.put(db.getName(), db); - } public boolean hasDocumentDB(String name) { return documentDbs.containsKey(name); @@ -72,7 +128,19 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer * Derives the schemas from the application package. * Also stores the document names contained in the schemas. */ - public abstract void deriveFromSchemas(DeployState deployState); + public void deriveFromSchemas(DeployState deployState) { + for (SchemaInfo spec : schemas().values()) { + if (spec.fullSchema() instanceof DocumentOnlySchema) continue; // TODO verify if this special handling is necessary + String schemaName = spec.fullSchema().getName(); + var derived = new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode()); + documentDbs.put(schemaName, new DocumentDatabase(this, schemaName, derived)); + if (spec.getIndexMode() == SchemaInfo.IndexMode.STREAMING) { + var parent = (TreeConfigProducer<AnyConfigProducer>)getParent(); + documentDBProducerForStreaming.put(schemaName, new AttributesProducer(parent, schemaName, derived)); + legacyproxy.add(new LegacyStreamingProxy(parent, clusterName, schemaName, derived)); + } + } + } /** Returns the document databases contained in this cluster */ public List<DocumentDatabase> getDocumentDbs() { @@ -80,7 +148,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer } public String getClusterName() { return clusterName; } - public final String getIndexingModeName() { return getIndexingMode().getName(); } public final boolean hasStreaming() { return schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING); } @@ -91,7 +158,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public final void setQueryTimeout(Double to) { this.queryTimeout = to; } public final void setVisibilityDelay(double delay) { this.visibilityDelay = delay; } - protected abstract IndexingMode getIndexingMode(); public final Double getVisibilityDelay() { return visibilityDelay; } public final Double getQueryTimeout() { return queryTimeout; } public final void setClusterIndex(int index) { this.index = index; } @@ -100,13 +166,26 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) { DocumentDatabase db = documentDbs.get(documentType); if (db != null) { - fillDocumentDBConfig(db, builder); + builder.inputdoctypename(documentType); + if (db.getDerivedConfiguration().isStreaming()) { + builder.configid(documentDBProducerForStreaming.get(documentType).getConfigId()); + } else { + builder.configid(db.getConfigId()); + } } } - protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { - ddbB.inputdoctypename(sdoc.getSchemaName()) - .configid(sdoc.getConfigId()); + public QrSearchersConfig.Searchcluster.Builder getQrSearcherConfig() { + var builder = new QrSearchersConfig.Searchcluster.Builder() + .name(getClusterName()) + .rankprofiles_configid(getConfigId()) + .storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(getStorageRouteSpec())) + .indexingmode(hasStreaming() ? QrSearchersConfig.Searchcluster.Indexingmode.STREAMING + : QrSearchersConfig.Searchcluster.Indexingmode.REALTIME); + for (SchemaInfo spec : schemas().values()) { + builder.searchdef(spec.fullSchema().getName()); + } + return builder; } @Override @@ -139,27 +218,33 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer new Join(documentDbs.values()).getConfig(builder); } - @Override - public String toString() { return "search-capable cluster '" + clusterName + "'"; } - - public static final class IndexingMode { - - public static final IndexingMode REALTIME = new IndexingMode("REALTIME"); - public static final IndexingMode STREAMING = new IndexingMode("STREAMING"); - - private final String name; - - private IndexingMode(String name) { - this.name = name; + public void getConfig(ClusterConfig.Builder builder) { + builder.clusterId(getClusterIndex()); + builder.clusterName(getClusterName()); + builder.storageRoute(getClusterName()); + builder.configid(getConfigId()); + if (visibilityDelay != null) { + builder.cacheTimeout(convertVisibilityDelay(visibilityDelay)); } - - public String getName() { return name; } - - public String toString() { - return "indexingmode: " + name; + if (hasStreaming()) { + builder.indexMode(ClusterConfig.IndexMode.Enum.STREAMING); + } else { + builder.indexMode(ClusterConfig.IndexMode.Enum.INDEX); } } + // The semantics of visibility delay in search is deactivating caches if the + // delay is less than 1.0, in qrs the cache is deactivated if the delay is 0 + // (or less). 1.0 seems a little arbitrary, so just doing the conversion + // here instead of having two totally independent implementations having to + // follow each other down in the modules. + private static Double convertVisibilityDelay(Double visibilityDelay) { + return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay; + } + + @Override + public String toString() { return "search-capable cluster '" + clusterName + "'"; } + /** * Class used to retrieve combined configuration from multiple document databases. * It is not a direct {@link ConfigInstance.Producer} of those configs, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java deleted file mode 100644 index e188a086614..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.schema.Schema; -import com.yahoo.schema.derived.AttributeFields; -import com.yahoo.schema.derived.DerivedConfiguration; -import com.yahoo.schema.derived.SchemaInfo; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.SummaryConfig; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import com.yahoo.vespa.config.search.summary.JuniperrcConfig; -import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; -import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; - -/** - * A search cluster of type streaming. - * - * @author baldersheim - * @author vegardh - */ -public class StreamingSearchCluster extends SearchCluster implements - AttributesConfig.Producer, - RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - RankingExpressionsConfig.Producer, - OnnxModelsConfig.Producer, - JuniperrcConfig.Producer, - SummaryConfig.Producer, - VsmsummaryConfig.Producer, - VsmfieldsConfig.Producer -{ - private final String storageRouteSpec; - private final AttributesProducer attributesConfig; - private final String docTypeName; - - public StreamingSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, - String docTypeName, String storageRouteSpec) { - super(parent, clusterName, index); - attributesConfig = new AttributesProducer(parent, docTypeName); - this.docTypeName = docTypeName; - this.storageRouteSpec = storageRouteSpec; - } - - @Override - protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } - public final String getStorageRouteSpec() { return storageRouteSpec; } - - public String getDocTypeName() { return docTypeName; } - - public DerivedConfiguration derived() { return db().getDerivedConfiguration(); } - - @Override - public void deriveFromSchemas(DeployState deployState) { - if (schemas().isEmpty()) return; - if (schemas().size() > 1) throw new IllegalArgumentException("Only a single schema is supported, got " + schemas().size()); - - Schema schema = schemas().values().stream().findAny().get().fullSchema(); - if ( ! schema.getName().equals(docTypeName)) - throw new IllegalArgumentException("Document type name '" + docTypeName + - "' must be the same as the schema name '" + schema.getName() + "'"); - add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(deployState, schema, SchemaInfo.IndexMode.STREAMING))); - } - - protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { - super.fillDocumentDBConfig(sdoc, ddbB); - ddbB.configid(attributesConfig.getConfigId()); // Temporary until fully cleaned up - } - - private DocumentDatabase db() { return getDocumentDbs().get(0); } - - // These are temporary until backend uses correct config id. - @Override public void getConfig(SummaryConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankingConstantsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(JuniperrcConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(VsmfieldsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(VsmsummaryConfig.Builder builder) { db().getConfig(builder);} - - private class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { - - AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType) { - super(parent, docType); - } - - @Override - public void getConfig(AttributesConfig.Builder builder) { - derived().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index b1d90b07b9d..5ead9812b56 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -2480,7 +2480,7 @@ public class ModelProvisioningTest { private long protonMemorySize(ContentCluster cluster) { ProtonConfig.Builder b = new ProtonConfig.Builder(); - cluster.getSearch().getIndexed().getSearchNode(0).getConfig(b); + cluster.getSearch().getSearchCluster().getSearchNode(0).getConfig(b); return b.build().hwinfo().memory().size(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java index d3e62dae947..2879baf7b88 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java @@ -29,7 +29,7 @@ public class StreamingValidatorTest { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/document_references_validation/") .create(); }); - assertTrue(exception.getMessage().contains("For search cluster 'content.ad', streaming schema 'ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " + + assertTrue(exception.getMessage().contains("For search cluster 'content', streaming schema 'ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " + "Document references and imported fields are not allowed in streaming search.")); } @@ -52,7 +52,7 @@ public class StreamingValidatorTest { "attribute { distance-metric: euclidean } }"); var warnings = filter(logger.warnings); assertEquals(1, warnings.size()); - assertEquals("For search cluster 'content.test', streaming schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting", + assertEquals("For search cluster 'content', streaming schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting", warnings.get(0)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java index ee64ceb6969..28eabd18539 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java @@ -92,9 +92,9 @@ public class StreamingSchemaClusterChangeValidatorTest { private static final String ATTRIBUTE_INT_FIELD = "field f1 type int { indexing: attribute | summary }"; private static final String ATTRIBUTE_FAST_ACCESS_INT_FIELD = "field f1 type int { indexing: attribute | summary \n attribute: fast-access }"; private static final List<ServiceInfo> FOO_SERVICE = - List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/0", "null")); + List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null")); private static final List<ServiceInfo> BAR_SERVICE = - List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/0", "null")); + List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null")); @Test void changing_field_type_requires_refeed() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index e531087ebd6..91ba97817d6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -44,7 +44,7 @@ public abstract class ContentClusterFixture { } protected DocumentDatabase currentDb() { - return currentCluster.getSearch().getIndexed().getDocumentDbs().get(0); + return currentCluster.getSearch().getSearchCluster().getDocumentDbs().get(0); } protected NewDocumentType currentDocType() { @@ -52,7 +52,7 @@ public abstract class ContentClusterFixture { } protected DocumentDatabase nextDb() { - return nextCluster.getSearch().getIndexed().getDocumentDbs().get(0); + return nextCluster.getSearch().getSearchCluster().getDocumentDbs().get(0); } protected NewDocumentType nextDocType() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 440e7a92ea1..fac82f3e675 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -62,8 +62,7 @@ public class ContentBuilderTest extends DomBuilderTest { "</content>"); ContentSearchCluster s = a.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); + assertFalse(s.hasSearchCluster()); assertTrue(a.getPersistence() instanceof com.yahoo.vespa.model.content.engines.DummyPersistence.Factory); } @@ -82,8 +81,7 @@ public class ContentBuilderTest extends DomBuilderTest { "</content>"); ContentSearchCluster s = a.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); + assertFalse(s.hasSearchCluster()); assertTrue(a.getPersistence() instanceof ProtonEngine.Factory); @@ -107,8 +105,7 @@ public class ContentBuilderTest extends DomBuilderTest { "</content>"); ContentSearchCluster s = a.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); + assertFalse(s.hasSearchCluster()); assertTrue(a.getPersistence() instanceof ProtonEngine.Factory); @@ -129,9 +126,8 @@ public class ContentBuilderTest extends DomBuilderTest { "</content>"); ContentSearchCluster s = a.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); - assertNull(s.getIndexed()); + assertFalse(s.hasSearchCluster()); + assertNull(s.getSearchCluster()); assertNull(a.getRootGroup().getName()); assertNull(a.getRootGroup().getIndex()); @@ -150,11 +146,10 @@ public class ContentBuilderTest extends DomBuilderTest { ContentCluster c = CollectionUtil.first(m.getContentClusters().values()); ContentSearchCluster s = c.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertEquals(1, s.getClusters().size()); - assertNotNull(s.getIndexed()); - assertEquals("clu", s.getIndexed().getClusterName()); - assertEquals(7.3, s.getIndexed().getQueryTimeout(), 0.0); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); + assertEquals("clu", s.getSearchCluster().getClusterName()); + assertEquals(7.3, s.getSearchCluster().getQueryTimeout(), 0.0); assertTrue(c.getPersistence() instanceof ProtonEngine.Factory); assertEquals(1, c.getStorageCluster().getChildren().size()); @@ -175,7 +170,7 @@ public class ContentBuilderTest extends DomBuilderTest { String services = getServices("<node hostalias='mockhost' distribution-key='0'/>" + "<node hostalias='mockhost' distribution-key='1'/>"); VespaModel m = new VespaModelCreatorWithMockPkg(createAppWithMusic(getHosts(), services)).create(); - IndexedSearchCluster sc = m.getContentClusters().get("clu").getSearch().getIndexed(); + IndexedSearchCluster sc = m.getContentClusters().get("clu").getSearch().getSearchCluster(); assertEquals(2, sc.getSearchNodeCount()); } @@ -196,11 +191,9 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = cluster.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertEquals(1, s.getClusters().size()); - assertNull(s.getIndexed()); - SearchCluster sc = s.getClusters().get(musicClusterId + ".music"); - assertEquals(musicClusterId + ".music", sc.getClusterName()); + assertTrue(s.hasSearchCluster()); + SearchCluster sc = s.getSearchCluster(); + assertEquals(musicClusterId, sc.getClusterName()); assertEquals(musicClusterId, sc.getStorageRouteSpec()); assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory); @@ -236,24 +229,11 @@ public class ContentBuilderTest extends DomBuilderTest { " <node hostalias=\"mockhost\" distribution-key=\"0\"/>" + " </group>" + "</content>"); - ContentSearchCluster s; - - s = cluster.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertEquals(2, s.getClusters().size()); - assertNull(s.getIndexed()); - { - String id = musicClusterId + ".book"; - SearchCluster sc = s.getClusters().get(id); - assertEquals(id, sc.getClusterName()); - assertEquals(musicClusterId, sc.getStorageRouteSpec()); - } - { - String id = musicClusterId + ".music"; - SearchCluster sc = s.getClusters().get(id); - assertEquals(id, sc.getClusterName()); - assertEquals(musicClusterId, sc.getStorageRouteSpec()); - } + ContentSearchCluster s = cluster.getSearch(); + assertTrue(s.hasSearchCluster()); + assertEquals(2, s.getSearchCluster().getDocumentDbs().size()); + assertTrue(s.getSearchCluster().hasDocumentDB("book")); + assertTrue(s.getSearchCluster().hasDocumentDB("music")); assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory); assertEquals(1, cluster.getStorageCluster().getChildren().size()); @@ -276,10 +256,9 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertEquals(1, s.getClusters().size()); - assertNotNull(s.getIndexed()); - assertEquals("b", s.getIndexed().getClusterName()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); + assertEquals("b", s.getSearchCluster().getClusterName()); assertTrue(b.getPersistence() instanceof ProtonEngine.Factory); assertEquals(1, b.getStorageCluster().getChildren().size()); @@ -308,8 +287,8 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(2, b.getStorageCluster().getChildren().size()); assertTrue(b.getRootGroup().getMmapNoCoreLimit().isPresent()); assertEquals(200000, b.getRootGroup().getMmapNoCoreLimit().get().longValue()); @@ -334,8 +313,8 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(1, b.getStorageCluster().getChildren().size()); assertEquals(1, s.getSearchNodes().size()); @@ -363,8 +342,8 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2"))); ContentSearchCluster s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(1, b.getStorageCluster().getChildren().size()); assertEquals(1, s.getSearchNodes().size()); @@ -389,8 +368,8 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(2, b.getStorageCluster().getChildren().size()); assertTrue(b.getRootGroup().getCoreOnOOM().isPresent()); assertTrue(b.getRootGroup().getCoreOnOOM().get()); @@ -414,8 +393,8 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); ContentSearchCluster s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(2, b.getStorageCluster().getChildren().size()); assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); @@ -438,8 +417,8 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); ContentSearchCluster s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(2, b.getStorageCluster().getChildren().size()); assertFalse(b.getRootGroup().getMmapNoCoreLimit().isPresent()); @@ -462,8 +441,8 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); ContentSearchCluster s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(2, b.getStorageCluster().getChildren().size()); assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); @@ -488,8 +467,8 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); ContentSearchCluster s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(4, b.getStorageCluster().getChildren().size()); assertTrue(b.getRootGroup().getNoVespaMalloc().isPresent()); assertEquals("proton", b.getRootGroup().getNoVespaMalloc().get()); @@ -522,8 +501,8 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); ContentSearchCluster s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(4, b.getStorageCluster().getChildren().size()); assertFalse(b.getRootGroup().getNoVespaMalloc().isPresent()); assertFalse(b.getRootGroup().getVespaMalloc().isPresent()); @@ -553,8 +532,8 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(2, b.getStorageCluster().getChildren().size()); assertTrue(b.getStorageCluster().getChildren().get("0").getAffinity().isPresent()); assertEquals(0, b.getStorageCluster().getChildren().get("0").getAffinity().get().cpuSocket()); @@ -588,8 +567,8 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = b.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertNotNull(s.getIndexed()); + assertTrue(s.hasSearchCluster()); + assertNotNull(s.getSearchCluster()); assertEquals(6, b.getStorageCluster().getChildren().size()); assertTrue(b.getRootGroup().useCpuSocketAffinity()); @@ -627,18 +606,17 @@ public class ContentBuilderTest extends DomBuilderTest { void requireBug5357273() { try { createContent( - " <content version='1.0' id='storage'>\n" + - " <redundancy>3</redundancy>\n" + - " <documents>" + - " <document type='music' mode='index'/>" + - " </documents>" + - " <group>\n" + - " <node hostalias='mockhost' distribution-key='0' />\n" + - " </group>\n" + - " <engine>\n" + - " <dummy/>\n" + - " </engine>\n" + - " </content>\n"); + """ + <content version='1.0' id='storage'> + <redundancy>3</redundancy> + <documents> <document type='music' mode='index'/> </documents> <group> + <node hostalias='mockhost' distribution-key='0' /> + </group> + <engine> + <dummy/> + </engine> + </content> + """); fail(); } catch (Exception e) { @@ -716,8 +694,8 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); ProtonConfig.Builder builder = new ProtonConfig.Builder(); - content.getSearch().getIndexed().getSearchNode(0).cascadeConfig(builder); - content.getSearch().getIndexed().getSearchNode(0).addUserConfig(builder); + content.getSearch().getSearchCluster().getSearchNode(0).cascadeConfig(builder); + content.getSearch().getSearchCluster().getSearchNode(0).addUserConfig(builder); ProtonConfig config = new ProtonConfig(builder); assertEquals(1, config.search().mmap().options().size()); assertEquals(ProtonConfig.Search.Mmap.Options.POPULATE, config.search().mmap().options(0)); @@ -847,8 +825,8 @@ public class ContentBuilderTest extends DomBuilderTest { private ProtonConfig getProtonConfig(ContentCluster content) { ProtonConfig.Builder configBuilder = new ProtonConfig.Builder(); - content.getSearch().getIndexed().getSearchNode(0).cascadeConfig(configBuilder); - content.getSearch().getIndexed().getSearchNode(0).addUserConfig(configBuilder); + content.getSearch().getSearchCluster().getSearchNode(0).cascadeConfig(configBuilder); + content.getSearch().getSearchCluster().getSearchNode(0).addUserConfig(configBuilder); return new ProtonConfig(configBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 8f02cf7a6d4..02a7a946e17 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; @@ -28,18 +27,13 @@ public class MockSearchClusters { @Override public void deriveFromSchemas(DeployState deployState) { } - @Override public List<DocumentDatabase> getDocumentDbs() {return List.of();} + @Override public List<DocumentDatabase> getDocumentDbs() { return List.of(); } @Override public void getConfig(AttributesConfig.Builder builder) {} @Override public void getConfig(DocumentdbInfoConfig.Builder builder) {} @Override public void getConfig(IndexInfoConfig.Builder builder) {} @Override public void getConfig(IlscriptsConfig.Builder builder) {} @Override public void getConfig(SchemaInfoConfig.Builder builder) {} - @Override - protected SearchCluster.IndexingMode getIndexingMode() { - return streaming ? SearchCluster.IndexingMode.STREAMING : SearchCluster.IndexingMode.REALTIME; - } - } public static SearchCluster mockSearchCluster(AbstractConfigProducerRoot root, String clusterName, int clusterIndex, boolean isStreaming) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index 6f9127779db..e9e96d8b0cf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -113,7 +113,7 @@ public class IndexedTest extends ContentBaseTest { VespaModelCreatorWithMockPkg creator = getIndexedVespaModelCreator(); VespaModel model = creator.create(); DeployState deployState = creator.deployState; - IndexedSearchCluster cluster = model.getContentClusters().get("test").getSearch().getIndexed(); + IndexedSearchCluster cluster = model.getContentClusters().get("test").getSearch().getSearchCluster(); assertEquals(3, cluster.getDocumentDbs().size()); NewDocumentType type1 = deployState.getDocumentModel().getDocumentManager().getDocumentType("type1"); NewDocumentType type2 = deployState.getDocumentModel().getDocumentManager().getDocumentType("type2"); @@ -163,7 +163,6 @@ public class IndexedTest extends ContentBaseTest { // "transactionlogserver"}; // DomContentBuilderTest.assertServices(h, expectedServices); ContentCluster s = model.getContentClusters().get("test"); - assertFalse(s.getSearch().hasIndexedCluster()); StorServerConfig.Builder builder = new StorServerConfig.Builder(); s.getStorageCluster().getConfig(builder); @@ -175,7 +174,6 @@ public class IndexedTest extends ContentBaseTest { VespaModel model = getStreamingVespaModel(); ContentCluster s = model.getContentClusters().get("test"); assertNotNull(s); - assertFalse(s.getSearch().hasIndexedCluster()); ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID); assertEquals(1, config.storage().size()); assertEquals("test", config.storage(0).name()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 0cdcf0534c2..8b83c941631 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -30,7 +30,7 @@ public class ClusterTest { " <query-timeout>1.1</query-timeout>", " <visibility-delay>2.3</visibility-delay>", "</search>")); - SearchCluster searchCluster = cluster.getSearch().getIndexed(); + SearchCluster searchCluster = cluster.getSearch().getSearchCluster(); assertNotNull(searchCluster); assertEquals(1.1, searchCluster.getQueryTimeout(), DELTA); assertEquals(2.3, searchCluster.getVisibilityDelay(), DELTA); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index d3741405c15..b4c625a599b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -309,7 +309,7 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); var model = tester.createModelWithMode(mode, sds); - DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); + DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); assertEquals(1, dcfg.documentdb().size()); DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0); assertEquals("type", db.name()); @@ -328,13 +328,11 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); var model = tester.createModel(sds, ""); DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); - assertEquals(1, indexed_cfg.documentdb().size()); + assertEquals(2, indexed_cfg.documentdb().size()); var db = indexed_cfg.documentdb(0); assertEquals("a", db.name()); assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode()); - DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b"); - assertEquals(1, streaming_cfg.documentdb().size()); - db = streaming_cfg.documentdb(0); + db = indexed_cfg.documentdb(1); assertEquals("b", db.name()); assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java index d3c026dbc0d..1b0b41a8d12 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java @@ -9,6 +9,7 @@ import com.yahoo.slime.Type; import static com.yahoo.config.model.api.EndpointCertificateMetadata.Provider.digicert; import static com.yahoo.config.model.api.EndpointCertificateMetadata.Provider.globalsign; +import static com.yahoo.config.model.api.EndpointCertificateMetadata.Provider.letsencrypt; import static com.yahoo.config.model.api.EndpointCertificateMetadata.Provider.zerossl; /** @@ -53,6 +54,7 @@ public class EndpointCertificateMetadataSerializer { case "digicert" -> digicert; case "globalsign" -> globalsign; case "zerossl" -> zerossl; + case "letsencrypt" -> letsencrypt; default -> digicert; }; } @@ -62,6 +64,8 @@ public class EndpointCertificateMetadataSerializer { case digicert -> "digicert"; case globalsign -> "globalsign"; case zerossl -> "zerossl"; + case letsencrypt -> "letsencrypt"; }; } + } diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index ac0c0dd6ada..7cda25a21d8 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -23,10 +23,6 @@ com.yahoo.prelude.searcher.JuniperSearcher.defaultdoctype string default="" com.yahoo.prelude.searcher.XMLStringSearcher.source string default="" -## Default docsum class the QR server should ask the backend to -## use for representing hints as default. -com.yahoo.prelude.fastsearch.IndexedBackend.docsum.defaultclass string default="" - com.yahoo.prelude.querytransform.PhrasingSearcher.automatonfile string default="" com.yahoo.prelude.querytransform.NonPhrasingSearcher.automatonfile string default="" com.yahoo.prelude.querytransform.TermReplacingSearcher.termlist[] string diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 73376ac4b25..18d1345cb06 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -2492,7 +2492,7 @@ "public" ], "methods" : [ - "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.search.config.ClusterConfig, com.yahoo.vespa.config.search.AttributesConfig)", + "public void <init>(com.yahoo.search.config.ClusterConfig, com.yahoo.vespa.config.search.AttributesConfig)", "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)" ], "fields" : [ diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 855a524473d..cb5ac7a6a4f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -6,7 +6,6 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.ClusterParams; @@ -71,7 +70,6 @@ public class ClusterSearcher extends Searcher { @Inject public ClusterSearcher(ComponentId id, Executor executor, - QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo, @@ -84,7 +82,6 @@ public class ClusterSearcher extends Searcher { this.schemaInfo = schemaInfo; int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); - QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); this.globalPhaseRanker = globalPhaseRanker; schema2Searcher = new LinkedHashMap<>(); @@ -92,9 +89,7 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); VespaBackend streaming = null, indexed = null; - ClusterParams clusterParams = makeClusterParams(searchClusterIndex, qrsConfig - .com().yahoo().prelude().fastsearch().IndexedBackend().docsum() - .defaultclass(), documentDbConfig, schemaInfo); + ClusterParams clusterParams = makeClusterParams(searchClusterIndex, documentDbConfig, schemaInfo); for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX) { if (indexed == null) { @@ -103,7 +98,7 @@ public class ClusterSearcher extends Searcher { schema2Searcher.put(docDb.name(), indexed); } else if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING) { if (streaming == null) { - streaming = streamingCluster(clusterParams, searchClusterConfig, access); + streaming = streamingCluster(clusterParams, clusterConfig, access); vipStatus.addToRotation(streaming.getName()); } schema2Searcher.put(docDb.name(), streaming); @@ -111,21 +106,10 @@ public class ClusterSearcher extends Searcher { } } - private static QrSearchersConfig.Searchcluster getSearchClusterConfigFromClusterName(QrSearchersConfig config, String name) { - for (QrSearchersConfig.Searchcluster searchCluster : config.searchcluster()) { - if (searchCluster.name().equals(name)) { - return searchCluster; - } - } - throw new IllegalStateException("No configured search cluster '" + name + "' among : " + - config.searchcluster().stream().map(QrSearchersConfig.Searchcluster::name).toList()); - } - - private static ClusterParams makeClusterParams(int searchclusterIndex, String defaultSummary, - DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo) + private static ClusterParams makeClusterParams(int searchclusterIndex, DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo) { return new ClusterParams("sc" + searchclusterIndex + ".num" + 0, UUID.randomUUID().toString(), - defaultSummary, documentDbConfig, schemaInfo); + null, documentDbConfig, schemaInfo); } private static IndexedBackend searchDispatch(ClusterParams clusterParams, @@ -140,11 +124,11 @@ public class ClusterSearcher extends Searcher { } private static StreamingBackend streamingCluster(ClusterParams clusterParams, - QrSearchersConfig.Searchcluster searchClusterConfig, + ClusterConfig clusterConfig, VespaDocumentAccess access) { - return new StreamingBackend(clusterParams, searchClusterConfig.rankprofiles_configid(), - access, searchClusterConfig.storagecluster().routespec()); + return new StreamingBackend(clusterParams, clusterConfig.configid(), + access, clusterConfig.storageRoute()); } /** Do not use, for internal testing purposes only. **/ diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java index a25cc30955e..96bf40195e9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.searcher; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -32,14 +31,12 @@ public class ValidateSortingSearcher extends Searcher { private Map<String, AttributesConfig.Attribute> attributeNames = null; private String clusterName = ""; - private final QrSearchersConfig.Searchcluster.Indexingmode.Enum indexingMode; + private final boolean enabled; - public ValidateSortingSearcher(QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, - AttributesConfig attributesConfig) { + public ValidateSortingSearcher(ClusterConfig clusterConfig, AttributesConfig attributesConfig) { initAttributeNames(attributesConfig); - var searchCluster = qrsConfig.searchcluster(clusterConfig.clusterId()); - setClusterName(searchCluster.name()); - indexingMode = searchCluster.indexingmode(); + setClusterName(clusterConfig.clusterName()); + enabled = clusterConfig.indexMode() != ClusterConfig.IndexMode.Enum.STREAMING; } public String getClusterName() { @@ -70,12 +67,10 @@ public class ValidateSortingSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { ErrorMessage e = validate(query); - if (indexingMode != QrSearchersConfig.Searchcluster.Indexingmode.STREAMING) { - if (e != null) { - Result r = new Result(query); - r.hits().addError(e); - return r; - } + if (enabled && e != null) { + Result r = new Result(query); + r.hits().addError(e); + return r; } return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java index d4e295fb51f..165469d59ee 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java @@ -8,7 +8,6 @@ import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.grouping.request.AttributeMapLookupValue; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -45,17 +44,12 @@ public class GroupingValidator extends Searcher { /** * Constructs a new instance of this searcher with the given component id and config. * - * @param qrsConfig The shared config for all searchers. * @param clusterConfig The config for the cluster that this searcher is deployed for. */ @Inject - public GroupingValidator(QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, - AttributesConfig attributesConfig) { - int clusterId = clusterConfig.clusterId(); - var searchCluster = qrsConfig.searchcluster(clusterId); - QrSearchersConfig.Searchcluster.Indexingmode.Enum indexingMode = searchCluster.indexingmode(); - enabled = (indexingMode != QrSearchersConfig.Searchcluster.Indexingmode.STREAMING); - clusterName = searchCluster.name(); + public GroupingValidator(ClusterConfig clusterConfig, AttributesConfig attributesConfig) { + enabled = (clusterConfig.indexMode() != ClusterConfig.IndexMode.Enum.STREAMING); + clusterName = clusterConfig.clusterName(); for (AttributesConfig.Attribute attr : attributesConfig.attribute()) { attributes.put(attr.name(), attr); } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java index 7761660d551..c5ffb6c02ea 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.searchers; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; diff --git a/container-search/src/main/resources/configdefinitions/search.config.cluster.def b/container-search/src/main/resources/configdefinitions/search.config.cluster.def index 2bd98f86ca8..ac88d5baa24 100644 --- a/container-search/src/main/resources/configdefinitions/search.config.cluster.def +++ b/container-search/src/main/resources/configdefinitions/search.config.cluster.def @@ -5,17 +5,11 @@ namespace=search.config #Note: Use clusterName where possible instead clusterId int default=0 -# Not used -cacheSize int default=1 - #Timeout for internal searcher cache. Entries older than this number #of seconds will be removed from cache. 0 or less means the cache is #disabled. cacheTimeout double default=0 -#Whether or not to try another TLD if the current request fails. -failoverToRemote bool default=false - #The name of the cluster (as specified in qr-searchers) clusterName string @@ -25,3 +19,12 @@ maxQueryTimeout double default=600 #The maximum query timeout allowed before disabling the backend query cache for the given query (default 10 seconds). #Note that the query timeout is used as the query cache timeout in the backend if enabled. maxQueryCacheTimeout double default=10 + +#The configid to use to get other clusterspecific configs. +configid string default="" + +#The mbus route to the backing content cluster +storageRoute string default="" + +#Indexing mode for this cluster. This is a temporary workaround until this is correctly resolved per schema all over. +indexMode enum {INDEX, STREAMING, STORE_ONLY} default = INDEX diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 4adbce3add9..1668edd081f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -488,7 +488,6 @@ public class ClusterSearcherTestCase { return new ClusterSearcher(new ComponentId("test-id"), new InThreadExecutorService(), - qrSearchersConfig.build(), clusterConfig.build(), documentDbConfig.build(), new SchemaInfo(List.of(schema.build()), List.of()), diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java index 7b0e1867650..b889119171e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.searcher.test; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.searcher.ValidateSortingSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -30,14 +29,10 @@ public class ValidateSortingSearcherTestCase { @SuppressWarnings("deprecation") public ValidateSortingSearcherTestCase() { - QrSearchersConfig.Builder qrsCfg = new QrSearchersConfig.Builder(); - qrsCfg.searchcluster(new QrSearchersConfig.Searchcluster.Builder().name("giraffes")); - ClusterConfig.Builder clusterCfg = new ClusterConfig.Builder(). - clusterId(0). - clusterName("test"); + ClusterConfig.Builder clusterCfg = new ClusterConfig.Builder() + .clusterName("giraffes"); String attributesCfg = "file:src/test/java/com/yahoo/prelude/searcher/test/validate_sorting.cfg"; - searcher = new ValidateSortingSearcher(new QrSearchersConfig(qrsCfg), - new ClusterConfig(clusterCfg), + searcher = new ValidateSortingSearcher(new ClusterConfig(clusterCfg), ConfigGetter.getConfig(AttributesConfig.class, attributesCfg)); } @@ -82,7 +77,6 @@ public class ValidateSortingSearcherTestCase { return transform(QueryTestCase.httpEncode(sorting), null); } - @SuppressWarnings("deprecation") private String transform(String sorting, String language) { String q = "/?query=a"; if (sorting != null) { diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java index fbab022c084..edce8112f21 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.search.grouping; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.search.Query; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.grouping.request.GroupingOperation; @@ -191,17 +190,11 @@ public class GroupingValidatorTestCase { } private static void validateGrouping(String clusterName, AttributesConfig attributesConfig, Query query) { - QrSearchersConfig.Builder qrsConfig = new QrSearchersConfig.Builder().searchcluster( - new QrSearchersConfig.Searchcluster.Builder() - .indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.REALTIME) - .name(clusterName)); - ClusterConfig.Builder clusterConfig = new ClusterConfig.Builder(). - clusterId(0). - clusterName("test"); - new Execution( - new GroupingValidator(new QrSearchersConfig(qrsConfig), - new ClusterConfig(clusterConfig), - attributesConfig), - Execution.Context.createContextStub()).search(query); + ClusterConfig.Builder clusterConfig = new ClusterConfig.Builder() + .clusterName(clusterName) + .indexMode(ClusterConfig.IndexMode.Enum.INDEX); + new Execution(new GroupingValidator(clusterConfig.build(), attributesConfig), + Execution.Context.createContextStub()) + .search(query); } } diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 6ef0e26bade..9c757e666a2 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -166,7 +166,7 @@ <maven-deploy-plugin.vespa.version>3.1.1</maven-deploy-plugin.vespa.version> <maven-enforcer-plugin.vespa.version>3.4.1</maven-enforcer-plugin.vespa.version> <maven-failsafe-plugin.vespa.version>3.2.5</maven-failsafe-plugin.vespa.version> - <maven-gpg-plugin.vespa.version>3.2.0</maven-gpg-plugin.vespa.version> + <maven-gpg-plugin.vespa.version>3.2.1</maven-gpg-plugin.vespa.version> <maven-install-plugin.vespa.version>3.1.1</maven-install-plugin.vespa.version> <maven-jar-plugin.vespa.version>3.3.0</maven-jar-plugin.vespa.version> <maven-javadoc-plugin.vespa.version>3.6.3</maven-javadoc-plugin.vespa.version> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 4134ea337ab..8646121bd4b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -33,7 +33,6 @@ public class Cluster { private final ClusterResources min, max; private final IntRange groupSize; private final boolean required; - private final Autoscaling suggested; private final List<Autoscaling> suggestions; private final Autoscaling target; private final ClusterInfo clusterInfo; @@ -48,7 +47,6 @@ public class Cluster { ClusterResources maxResources, IntRange groupSize, boolean required, - Autoscaling suggested, List<Autoscaling> suggestions, Autoscaling target, ClusterInfo clusterInfo, @@ -60,7 +58,6 @@ public class Cluster { this.max = Objects.requireNonNull(maxResources); this.groupSize = Objects.requireNonNull(groupSize); this.required = required; - this.suggested = Objects.requireNonNull(suggested); this.suggestions = Objects.requireNonNull(suggestions); Objects.requireNonNull(target); if (target.resources().isPresent() && ! target.resources().get().isWithin(minResources, maxResources)) @@ -100,12 +97,6 @@ public class Cluster { public Autoscaling target() { return target; } /** - * The suggested resources, which may or may not be within the min and max limits, - * or empty if there is currently no recorded suggestion. - */ - public Autoscaling suggested() { return suggested; } - - /** * The list of suggested resources, which may or may not be within the min and max limits, * or empty if there is currently no recorded suggestion. * List is sorted by preference @@ -143,23 +134,19 @@ public class Cluster { public Cluster withConfiguration(boolean exclusive, Capacity capacity) { return new Cluster(id, exclusive, capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(), - suggested, suggestions, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); - } - - public Cluster withSuggested(Autoscaling suggested) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + suggestions, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); } public Cluster withSuggestions(List<Autoscaling> suggestions) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster withTarget(Autoscaling target) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster with(BcpGroupInfo bcpGroupInfo) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } /** Add or update (based on "at" time) a scaling event */ @@ -173,7 +160,7 @@ public class Cluster { scalingEvents.add(scalingEvent); prune(scalingEvents); - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } @Override @@ -205,7 +192,7 @@ public class Cluster { public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) { return new Cluster(id, exclusive, requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(), - Autoscaling.empty(), List.of(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); + List.of(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); } /** The predicted time it will take to rescale this cluster. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index fa1be83dbcf..07cfce79f4b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -98,8 +98,7 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { Mutex lock) { Optional<Cluster> cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; - applications().put(application.with(cluster.get().withSuggestions(suggestions) - .withSuggested(suggestions.stream().findFirst().orElse(Autoscaling.empty()))), lock); + applications().put(application.with(cluster.get().withSuggestions(suggestions)), lock); } private boolean isHigher(ClusterResources r1, ClusterResources r2) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 2dea70825ee..1315207efb8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -141,8 +141,8 @@ public class ApplicationSerializer { toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); toSlime(cluster.groupSize(), clusterObject.setObject(groupSizeKey)); clusterObject.setBool(requiredKey, cluster.required()); - // TODO(olaa): Remove 'suggested' once API clients migrate to suggestion list - toSlime(cluster.suggested(), clusterObject.setObject(suggestedKey)); + // TODO(olaa): Remove 'suggested' once all configservers have stopped reading entry + toSlime(Autoscaling.empty(), clusterObject.setObject(suggestedKey)); toSlime(cluster.suggestions(), clusterObject.setArray(suggestionsKey)); toSlime(cluster.target(), clusterObject.setObject(targetKey)); if (! cluster.clusterInfo().isEmpty()) @@ -159,7 +159,6 @@ public class ApplicationSerializer { clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), intRangeFromSlime(clusterObject.field(groupSizeKey)), clusterObject.field(requiredKey).asBool(), - autoscalingFromSlime(clusterObject.field(suggestedKey)), suggestionsFromSlime(clusterObject.field(suggestionsKey)), autoscalingFromSlime(clusterObject.field(targetKey)), clusterInfoFromSlime(clusterObject.field(clusterInfoKey)), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 0285e72a8a4..2bba0ee56be 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -79,9 +79,7 @@ public class ApplicationSerializer { toSlime(cluster.groupSize(), clusterObject.setObject("groupSize")); toSlime(currentResources, clusterObject.setObject("current")); if (cluster.shouldSuggestResources(currentResources)) { - toSlime(cluster.suggested(), clusterObject.setObject("suggested")); toSlime(cluster.suggestions(), clusterObject.setArray("suggestions")); - } toSlime(cluster.target(), clusterObject.setObject("target")); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index e7c9d1079fb..9d9771b3b0f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -225,14 +225,6 @@ public class MockNodeRepository extends NodeRepository { null), app1Id, provisioner); Application app1 = applications().get(app1Id).get(); Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); - cluster1 = cluster1.withSuggested(new Autoscaling(Autoscaling.Status.unavailable, - "", - Optional.of(new ClusterResources(6, 2, - new NodeResources(3, 20, 100, 1))), - clock().instant(), - Load.zero(), - Load.zero(), - Autoscaling.Metrics.zero())); cluster1 = cluster1.withSuggestions(List.of(new Autoscaling(Autoscaling.Status.unavailable, "", Optional.of(new ClusterResources(6, 2, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 90af6dca090..f25d4cc3c30 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -40,7 +40,6 @@ public class ApplicationSerializerTest { new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), IntRange.empty(), true, - Autoscaling.empty(), List.of(), Autoscaling.empty(), ClusterInfo.empty(), @@ -53,14 +52,6 @@ public class ApplicationSerializerTest { new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), IntRange.of(3, 5), false, - new Autoscaling(Autoscaling.Status.unavailable, - "", - Optional.of(new ClusterResources(20, 10, - new NodeResources(0.5, 4, 14, 16))), - Instant.ofEpochMilli(1234L), - new Load(0.1, 0.2, 0.3, 0.4, 0.5), - new Load(0.4, 0.5, 0.6, 0.7, 0.8), - new Autoscaling.Metrics(0.7, 0.8, 0.9)), List.of(new Autoscaling(Autoscaling.Status.unavailable, "", Optional.of(new ClusterResources(20, 10, @@ -106,7 +97,6 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); assertEquals(originalCluster.groupSize(), serializedCluster.groupSize()); assertEquals(originalCluster.required(), serializedCluster.required()); - assertEquals(originalCluster.suggested(), serializedCluster.suggested()); assertEquals(originalCluster.suggestions(), serializedCluster.suggestions()); assertEquals(originalCluster.target(), serializedCluster.target()); assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 6b6ef49fa5d..1f8178dff6a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -143,7 +143,6 @@ public class DynamicProvisioningTester { cluster.maxResources(), cluster.groupSize(), cluster.required(), - cluster.suggested(), cluster.suggestions(), cluster.target(), cluster.clusterInfo(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json index e74e705e1aa..9f73fcd9a46 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -45,43 +45,6 @@ "architecture":"x86_64" } }, - "suggested" : { - "status" : "unavailable", - "description" : "", - "resources" : { - "nodes": 6, - "groups": 2, - "resources": { - "vcpu": 3.0, - "memoryGb": 20.0, - "diskGb": 100.0, - "bandwidthGbps": 1.0, - "diskSpeed": "fast", - "storageType": "any", - "architecture": "any" - } - }, - "at" : 123, - "peak" : { - "cpu" : 0.0, - "memory" : 0.0, - "disk" : 0.0, - "gpu": 0.0, - "gpuMemory": 0.0 - }, - "ideal" : { - "cpu" : 0.0, - "memory" : 0.0, - "disk" : 0.0, - "gpu": 0.0, - "gpuMemory": 0.0 - }, - "metrics" : { - "queryRate" : 0.0, - "growthRateHeadroom" : 0.0, - "cpuCostPerQuery" : 0.0 - } - }, "suggestions": [ { "at": 123, |