// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
import com.yahoo.component.ComponentId;
import com.yahoo.container.logging.AccessLog;
import com.yahoo.container.logging.JSONAccessLog;
import com.yahoo.messagebus.routing.Hop;
import com.yahoo.messagebus.routing.HopBlueprint;
import com.yahoo.messagebus.routing.PolicyDirective;
import com.yahoo.messagebus.routing.Route;
import com.yahoo.messagebus.routing.RoutingTable;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
import com.yahoo.vespa.model.container.docproc.DocprocChain;
import com.yahoo.vespa.model.routing.DocumentProtocol;
import com.yahoo.vespa.model.routing.Protocol;
import com.yahoo.vespa.model.routing.Routing;
import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
/**
* @author Einar M R Rosenvinge
*/
public class IndexingAndDocprocRoutingTest extends ContentBaseTest {
@Test
void oneContentOneDoctypeImplicitIndexingClusterImplicitIndexingChain() {
final String CLUSTERNAME = "musiccluster";
SearchClusterSpec searchCluster = new SearchClusterSpec(CLUSTERNAME, null, null);
searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
VespaModel model = getIndexedContentVespaModel(List.of(), List.of(searchCluster));
assertIndexing(model, new DocprocClusterSpec("container", new DocprocChainSpec("container/chain.indexing")));
assertFeedingRoute(model, CLUSTERNAME, "container/chain.indexing");
}
@Test
void oneContentTwoDoctypesImplicitIndexingClusterImplicitIndexingChain() {
final String CLUSTERNAME = "musicandbookscluster";
SearchClusterSpec searchCluster = new SearchClusterSpec(CLUSTERNAME, null, null);
searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
searchCluster.searchDefs.add(new SearchDefSpec("book", "author", "title"));
VespaModel model = getIndexedContentVespaModel(List.of(), List.of(searchCluster));
assertIndexing(model, new DocprocClusterSpec("container", new DocprocChainSpec("container/chain.indexing")));
assertFeedingRoute(model, CLUSTERNAME, "container/chain.indexing");
}
@Test
void twoContentTwoDoctypesImplicitIndexingClusterImplicitIndexingChain() {
final String MUSIC = "musiccluster";
SearchClusterSpec musicCluster = new SearchClusterSpec(MUSIC, null, null);
musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
final String BOOKS = "bookscluster";
SearchClusterSpec booksCluster = new SearchClusterSpec(BOOKS, null, null);
booksCluster.searchDefs.add(new SearchDefSpec("book", "author", "title"));
VespaModel model = getIndexedContentVespaModel(List.of(), List.of(musicCluster, booksCluster));
assertIndexing(model,
new DocprocClusterSpec("container", new DocprocChainSpec("container/chain.indexing")));
assertFeedingRoute(model, MUSIC, "container/chain.indexing");
assertFeedingRoute(model, BOOKS, "container/chain.indexing");
}
@Test
void oneContentOneDoctypeExplicitIndexingClusterImplicitIndexingChain() {
final String CLUSTERNAME = "musiccluster";
SearchClusterSpec searchCluster = new SearchClusterSpec(CLUSTERNAME, "dpcluster", null);
searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
VespaModel model = getIndexedContentVespaModel(List.of(new DocprocClusterSpec("dpcluster")), List.of(searchCluster));
assertIndexing(model, new DocprocClusterSpec("dpcluster", new DocprocChainSpec("dpcluster/chain.indexing")));
assertFeedingRoute(model, CLUSTERNAME, "dpcluster/chain.indexing");
}
@Test
void oneSearchOneDoctypeExplicitIndexingClusterExplicitIndexingChain() {
String xml =
"\n" +
"\n" +
" \n" +
" \n" +
" \n" +
"\n" +
" \n" +
" 2\n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
"\n";
VespaModel model = getIndexedSearchVespaModel(xml);
assertIndexing(model, new DocprocClusterSpec("dpcluster", new DocprocChainSpec("dpcluster/chain.fooindexing", "indexing"),
new DocprocChainSpec("dpcluster/chain.indexing")));
assertFeedingRouteIndexed(model, "searchcluster", "dpcluster/chain.fooindexing");
}
@Test
void twoContentTwoDoctypesExplicitIndexingInSameIndexingCluster() {
final String MUSIC = "musiccluster";
SearchClusterSpec musicCluster = new SearchClusterSpec(MUSIC, "dpcluster", null);
musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
final String BOOKS = "bookscluster";
SearchClusterSpec booksCluster = new SearchClusterSpec(BOOKS, "dpcluster", null);
booksCluster.searchDefs.add(new SearchDefSpec("book", "author", "title"));
VespaModel model = getIndexedContentVespaModel(List.of(new DocprocClusterSpec("dpcluster")),
List.of(musicCluster, booksCluster));
assertIndexing(model, new DocprocClusterSpec("dpcluster", new DocprocChainSpec("dpcluster/chain.indexing")));
assertFeedingRoute(model, MUSIC, "dpcluster/chain.indexing");
assertFeedingRoute(model, BOOKS, "dpcluster/chain.indexing");
}
@Test
void noContentClustersOneDocprocCluster() {
String services =
"\n" +
"\n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
"\n";
List sds = ApplicationPackageUtils.generateSchemas("music", "title", "artist");
VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create();
assertIndexing(model, new DocprocClusterSpec("dokprok"));
}
@Test
void twoContentTwoDoctypesExplicitIndexingInDifferentIndexingClustersExplicitChain() {
final String MUSIC = "musiccluster";
SearchClusterSpec musicCluster = new SearchClusterSpec(MUSIC, "dpmusiccluster", "dpmusicchain");
musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
final String BOOKS = "bookscluster";
SearchClusterSpec booksCluster = new SearchClusterSpec(BOOKS, "dpbookscluster", "dpbookschain");
booksCluster.searchDefs.add(new SearchDefSpec("book", "author", "title"));
DocprocClusterSpec dpMusicCluster = new DocprocClusterSpec("dpmusiccluster", new DocprocChainSpec("dpmusicchain", "indexing"));
DocprocClusterSpec dpBooksCluster = new DocprocClusterSpec("dpbookscluster", new DocprocChainSpec("dpbookschain", "indexing"));
VespaModel model = getIndexedContentVespaModel(List.of(dpMusicCluster, dpBooksCluster),
List.of(musicCluster, booksCluster));
//after we generated model, add indexing chains for validation:
dpMusicCluster.chains.clear();
dpMusicCluster.chains.add(new DocprocChainSpec("dpmusiccluster/chain.indexing"));
dpMusicCluster.chains.add(new DocprocChainSpec("dpmusiccluster/chain.dpmusicchain"));
dpBooksCluster.chains.clear();
dpBooksCluster.chains.add(new DocprocChainSpec("dpbookscluster/chain.indexing"));
dpBooksCluster.chains.add(new DocprocChainSpec("dpbookscluster/chain.dpbookschain"));
assertIndexing(model, dpMusicCluster, dpBooksCluster);
assertFeedingRoute(model, MUSIC, "dpmusiccluster/chain.dpmusicchain");
assertFeedingRoute(model, BOOKS, "dpbookscluster/chain.dpbookschain");
}
@Test
void requiresIndexingInheritance() {
try {
SearchClusterSpec musicCluster = new SearchClusterSpec("musiccluster",
"dpmusiccluster",
"dpmusicchain");
musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
DocprocClusterSpec dpMusicCluster = new DocprocClusterSpec("dpmusiccluster", new DocprocChainSpec("dpmusicchain"));
getIndexedContentVespaModel(List.of(dpMusicCluster), List.of(musicCluster));
fail("Expected exception");
}
catch (IllegalArgumentException e) {
assertEquals("Docproc chain 'dpmusicchain' must inherit from the 'indexing' chain", e.getMessage());
}
}
@Test
void indexingChainShouldNotBeTheDefaultChain() {
try {
SearchClusterSpec musicCluster = new SearchClusterSpec("musiccluster",
"dpmusiccluster",
"default");
musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album"));
DocprocClusterSpec dpMusicCluster = new DocprocClusterSpec("dpmusiccluster", new DocprocChainSpec("default", "indexing"));
getIndexedContentVespaModel(List.of(dpMusicCluster), List.of(musicCluster));
fail("Expected exception");
}
catch (IllegalArgumentException e) {
assertTrue(e.getMessage().startsWith("content cluster 'musiccluster' specifies the chain 'default' as indexing chain"));
}
}
private void assertIndexing(VespaModel model, DocprocClusterSpec... expectedDocprocClusters) {
Map docprocClusters = getDocprocClusters(model);
assertEquals(expectedDocprocClusters.length, docprocClusters.size());
for (DocprocClusterSpec expectedDocprocCluster : expectedDocprocClusters) {
ContainerCluster docprocCluster = docprocClusters.get(expectedDocprocCluster.name);
assertNotNull(docprocCluster);
assertEquals(expectedDocprocCluster.name, docprocCluster.getName());
ContainerDocproc containerDocproc = docprocCluster.getDocproc();
assertNotNull(containerDocproc);
List chains = containerDocproc.getChains().allChains().allComponents();
assertEquals(expectedDocprocCluster.chains.size(), chains.size());
List actualDocprocChains = new ArrayList<>();
for (DocprocChain chain : chains) {
actualDocprocChains.add(chain.getServiceName());
}
List expectedDocprocChainStrings = new ArrayList<>();
for (DocprocChainSpec spec : expectedDocprocCluster.chains) {
expectedDocprocChainStrings.add(spec.name);
}
assertTrue(actualDocprocChains.containsAll(expectedDocprocChainStrings));
assertNotNull(docprocCluster.getComponentsMap().get(ComponentId.fromString(AccessLog.class.getName())));
assertNotNull(docprocCluster.getComponentsMap().get(ComponentId.fromString(JSONAccessLog.class.getName())));
}
}
private Map getDocprocClusters(VespaModel model) {
Map docprocClusters = new HashMap<>();
for (ContainerCluster containerCluster : model.getContainerClusters().values()) {
if (containerCluster.getDocproc() != null) {
docprocClusters.put(containerCluster.getName(), containerCluster);
}
}
return docprocClusters;
}
private void assertFeedingRoute(VespaModel model, String searchClusterName, String indexingHopName) {
Routing routing = model.getRouting();
List protocols = routing.getProtocols();
DocumentProtocol documentProtocol = null;
for (Protocol protocol : protocols) {
if (protocol instanceof DocumentProtocol) {
documentProtocol = (DocumentProtocol) protocol;
}
}
assertNotNull(documentProtocol);
RoutingTable table = new RoutingTable(documentProtocol.getRoutingTableSpec());
HopBlueprint indexingHop = table.getHop("indexing");
assertNotNull(indexingHop);
assertEquals(1, indexingHop.getNumDirectives());
assertTrue(indexingHop.getDirective(0) instanceof PolicyDirective);
assertEquals("[DocumentRouteSelector]", indexingHop.getDirective(0).toString());
//assertThat(indexingHop.getNumRecipients(), is(1));
//assertThat(indexingHop.getRecipient(0).getServiceName(), is(searchClusterName));
Route route = table.getRoute(searchClusterName);
assertNotNull(route);
assertEquals(1, route.getNumHops());
Hop messageTypeHop = route.getHop(0);
assertEquals(1, messageTypeHop.getNumDirectives());
assertTrue(messageTypeHop.getDirective(0) instanceof PolicyDirective);
assertEquals("[MessageType:" + searchClusterName + "]", messageTypeHop.getDirective(0).toString());
PolicyDirective messageTypeDirective = (PolicyDirective) messageTypeHop.getDirective(0);
assertEquals("MessageType", messageTypeDirective.getName());
assertEquals(searchClusterName, messageTypeDirective.getParam());
String indexingRouteName = DocumentProtocol.getIndexedRouteName(model.getContentClusters().get(searchClusterName).getConfigId());
Route indexingRoute = table.getRoute(indexingRouteName);
assertEquals(2, indexingRoute.getNumHops());
assertEquals(indexingHopName, indexingRoute.getHop(0).getServiceName());
assertNotNull(indexingRoute.getHop(1));
}
private void assertFeedingRouteIndexed(VespaModel model, String searchClusterName, String indexingHopName) {
Routing routing = model.getRouting();
List protocols = routing.getProtocols();
DocumentProtocol documentProtocol = null;
for (Protocol protocol : protocols) {
if (protocol instanceof DocumentProtocol) {
documentProtocol = (DocumentProtocol) protocol;
}
}
assertNotNull(documentProtocol);
RoutingTable table = new RoutingTable(documentProtocol.getRoutingTableSpec());
Route indexingRoute = table.getRoute("searchcluster-index");
assertEquals(2, indexingRoute.getNumHops());
assertEquals(indexingHopName, indexingRoute.getHop(0).toString());
assertEquals("[Content:cluster=" + searchClusterName + "]", indexingRoute.getHop(1).toString());
}
private String createVespaServices(String mainPre, String contentClusterPre, String contentClusterPost,
String searchClusterPre, String searchClusterPost, String searchClusterPostPost,
String mainPost, List searchClusterSpecs) {
StringBuilder retval = new StringBuilder();
retval.append(mainPre);
for (SearchClusterSpec searchClusterSpec : searchClusterSpecs) {
retval.append(contentClusterPre).append(searchClusterSpec.name).append(contentClusterPost);
retval.append(searchClusterPre);
for (SearchDefSpec searchDefSpec : searchClusterSpec.searchDefs) {
retval.append(" \n");
}
if (searchClusterSpec.indexingClusterName != null) {
retval.append(" \n");
}
retval.append(searchClusterPost);
retval.append(searchClusterPostPost);
}
retval.append(mainPost);
System.err.println(retval);
return retval.toString();
}
private String createVespaServicesWithContent(List docprocClusterSpecs,
List searchClusterSpecs) {
String mainPre =
"\n" +
"\n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n" +
" \n";
int clusterNo = 0;
for (DocprocClusterSpec docprocClusterSpec : docprocClusterSpecs) {
String docprocCluster = "";
docprocCluster += " \n";
if (docprocClusterSpec.chains.size() > 0) {
docprocCluster += " \n";
for (DocprocChainSpec chain : docprocClusterSpec.chains) {
if (chain.inherits.isEmpty()) {
docprocCluster += " \n";
} else {
docprocCluster += " \n";
}
}
docprocCluster += " \n";
} else {
docprocCluster += " \n";
}
docprocCluster += " \n" +
" \n" +
" \n";
docprocCluster += " \n" +
" \n" +
" \n" +
" \n";
mainPre += docprocCluster;
clusterNo++;
}
String contentClusterPre =
" \n";
String searchClusterPre =
" 1\n" +
" \n";
String searchClusterPost =
" \n" +
" \n" +
" \n" +
" \n";
String searchClusterPostPost = " \n";
String mainPost =
"\n";
return createVespaServices(mainPre, contentClusterPre, contentClusterPost, searchClusterPre,
searchClusterPost, searchClusterPostPost, mainPost, searchClusterSpecs);
}
private VespaModel getIndexedSearchVespaModel(String xml) {
List sds = generateSchemas("music", "album", "artist");
return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
}
private VespaModel getIndexedContentVespaModel(List docprocClusterSpecs, List searchClusterSpecs) {
List sds = new ArrayList<>();
for (SearchClusterSpec cluster : searchClusterSpecs) {
for (SearchDefSpec def : cluster.searchDefs) {
sds.add(ApplicationPackageUtils.generateSchema(def.typeName, def.field1Name, def.field2Name));
}
}
return new VespaModelCreatorWithMockPkg(getHosts(),
createVespaServicesWithContent(docprocClusterSpecs, searchClusterSpecs), sds).create();
}
private static class SearchClusterSpec {
private final String name;
private final List searchDefs = new ArrayList<>(2);
private final String indexingClusterName;
private final String indexingChainName;
private SearchClusterSpec(String name, String indexingClusterName, String indexingChainName) {
this.name = name;
this.indexingClusterName = indexingClusterName;
this.indexingChainName = indexingChainName;
}
}
private static class SearchDefSpec {
private final String typeName;
private final String field1Name;
private final String field2Name;
private SearchDefSpec(String typeName, String field1Name, String field2Name) {
this.typeName = typeName;
this.field1Name = field1Name;
this.field2Name = field2Name;
}
}
private class DocprocClusterSpec {
private final String name;
private final List chains = new ArrayList<>();
private DocprocClusterSpec(String name, DocprocChainSpec ... chains) {
this.name = name;
this.chains.addAll(Arrays.asList(chains));
}
}
private static class DocprocChainSpec {
private final String name;
private final List inherits = new ArrayList<>();
private DocprocChainSpec(String name, String ... inherits) {
this.name = name;
this.inherits.addAll(Arrays.asList(inherits));
}
}
public static String generateSchema(String name, String field1, String field2) {
return "schema " + name + " {" +
" document " + name + " {" +
" field " + field1 + " type string {\n" +
" indexing: index | summary\n" +
" summary: dynamic\n" +
" }\n" +
" field " + field2 + " type int {\n" +
" indexing: attribute | summary\n" +
" attribute: fast-access\n" +
" }\n" +
" field " + field2 + "_nfa type int {\n" +
" indexing: attribute \n" +
" }\n" +
" }\n" +
" rank-profile staticrank inherits default {" +
" first-phase { expression: attribute(" + field2 + ") }" +
" }" +
" rank-profile summaryfeatures inherits default {" +
" first-phase { expression: attribute(" + field2 + ") }\n" +
" summary-features: attribute(" + field2 + ")" +
" }" +
" rank-profile inheritedsummaryfeatures inherits summaryfeatures {" +
" }" +
" rank-profile rankfeatures {" +
" first-phase { expression: attribute(" + field2 + ") }\n" +
" rank-features: attribute(" + field2 + ")" +
" }" +
" rank-profile inputs {" +
" inputs {" +
" query(foo) tensor(x[10])\n" +
" query(bar) tensor(key{},x[1000])" +
" }" +
" }" +
"}";
}
public static List generateSchemas(String ... sdNames) {
return generateSchemas(Arrays.asList(sdNames));
}
public static List generateSchemas(List sdNames) {
List sds = new ArrayList<>();
int i = 0;
for (String sdName : sdNames) {
sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2)));
i = i + 2;
}
return sds;
}
}