diff options
30 files changed, 518 insertions, 96 deletions
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java index 054e820b1a4..7c34539921b 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.Optional; - /** * Main entry point for class analysis * @@ -21,6 +20,7 @@ import java.util.Optional; * @author ollivir */ public class Analyze { + public static ClassFileMetaData analyzeClass(File classFile) { return analyzeClass(classFile, null); } @@ -44,7 +44,7 @@ public class Analyze { } static Optional<String> internalNameToClassName(String internalClassName) { - if(internalClassName == null) { + if (internalClassName == null) { return Optional.empty(); } else { return getClassName(Type.getObjectType(internalClassName)); @@ -53,12 +53,14 @@ public class Analyze { static Optional<String> getClassName(Type aType) { switch (aType.getSort()) { - case Type.ARRAY: - return getClassName(aType.getElementType()); - case Type.OBJECT: - return Optional.of(aType.getClassName()); - default: - return Optional.empty(); + case Type.ARRAY: + return getClassName(aType.getElementType()); + case Type.OBJECT: + return Optional.of(aType.getClassName()); + case Type.METHOD: + return getClassName(aType.getReturnType()); + default: + return Optional.empty(); } } @@ -89,4 +91,5 @@ public class Analyze { } }; } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java index 88b752ae945..307509f0452 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java @@ -24,8 +24,9 @@ import java.util.Set; * @author ollivir */ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { + private String name = null; - private Set<String> imports = new HashSet<>(); + private final Set<String> imports = new HashSet<>(); private Optional<ExportPackageAnnotation> exportPackageAnnotation = Optional.empty(); private final Optional<ArtifactVersion> defaultExportPackageVersion; @@ -168,4 +169,5 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { assert (!imports.contains("int")); return new ClassFileMetaData(name, imports, exportPackageAnnotation); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java index b4c44c9ed40..051df41d62b 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java @@ -14,6 +14,7 @@ import java.util.Set; * @author ollivir */ public class AnalyzeFieldVisitor extends FieldVisitor implements ImportCollector { + private final AnalyzeClassVisitor analyzeClassVisitor; private final Set<String> imports = new HashSet<>(); @@ -46,4 +47,5 @@ public class AnalyzeFieldVisitor extends FieldVisitor implements ImportCollector public void visitEnd() { analyzeClassVisitor.addImports(imports); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java index b7d1291d54a..7913f315cdd 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java @@ -20,7 +20,9 @@ import java.util.Set; * @author ollivir */ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { + private final Set<String> imports = new HashSet<>(); + private final AnalyzeClassVisitor analyzeClassVisitor; AnalyzeMethodVisitor(AnalyzeClassVisitor analyzeClassVisitor) { @@ -104,13 +106,14 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { @Override public void visitInvokeDynamicInsn(String name, String desc, Handle bootstrapMethod, Object... bootstrapMethodArgs) { + addImportWithTypeDesc(desc); for (Object arg : bootstrapMethodArgs) { if (arg instanceof Type) { addImport((Type) arg); } else if (arg instanceof Handle) { addImportWithInternalName(((Handle) arg).getOwner()); Arrays.asList(Type.getArgumentTypes(desc)).forEach(this::addImport); - } else if ((arg instanceof Number) == false && (arg instanceof String) == false) { + } else if ( ! (arg instanceof Number) && ! (arg instanceof String)) { throw new AssertionError("Unexpected type " + arg.getClass() + " with value '" + arg + "'"); } } @@ -165,4 +168,5 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { @Override public void visitCode() { } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java index d3c32e11201..61c37e99edf 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java @@ -12,10 +12,10 @@ import java.util.Set; * @author Tony Vaagenes * @author ollivir */ - class AnalyzeSignatureVisitor extends SignatureVisitor implements ImportCollector { + private final AnalyzeClassVisitor analyzeClassVisitor; - private Set<String> imports = new HashSet<>(); + private final Set<String> imports = new HashSet<>(); AnalyzeSignatureVisitor(AnalyzeClassVisitor analyzeClassVisitor) { super(Opcodes.ASM7); @@ -116,4 +116,5 @@ class AnalyzeSignatureVisitor extends SignatureVisitor implements ImportCollecto if (signature != null) new SignatureReader(signature).acceptType(new AnalyzeSignatureVisitor(analyzeClassVisitor)); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java index 4195f342e92..5601430a27f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ClassFileMetaData.java @@ -11,6 +11,7 @@ import java.util.Set; * @author ollivir */ public class ClassFileMetaData { + private final String name; private final Set<String> referencedClasses; private final Optional<ExportPackageAnnotation> exportPackage; @@ -32,4 +33,5 @@ public class ClassFileMetaData { public Optional<ExportPackageAnnotation> getExportPackage() { return exportPackage; } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java index 955056b2306..7f3fb9522f7 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ExportPackageAnnotation.java @@ -9,6 +9,7 @@ import java.util.regex.Pattern; * @author ollivir */ public class ExportPackageAnnotation { + private final int major; private final int minor; private final int micro; @@ -59,4 +60,5 @@ public class ExportPackageAnnotation { public int hashCode() { return Objects.hash(major, minor, micro, qualifier); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java index 1dc1e49897d..e341ef1a80f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/ImportCollector.java @@ -11,6 +11,7 @@ import java.util.Set; * @author ollivir */ public interface ImportCollector { + Set<String> imports(); default void addImportWithTypeDesc(String typeDescriptor) { @@ -32,4 +33,5 @@ public interface ImportCollector { default void addImport(Optional<String> anImport) { anImport.ifPresent(pkg -> imports().add(pkg)); } + } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java index 3d20bfd54df..9eef8a55c01 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Packages.java @@ -11,6 +11,7 @@ import java.util.Set; * @author ollivir */ public class Packages { + public static class PackageMetaData { public final Set<String> definedPackages; public final Set<String> referencedExternalPackages; @@ -40,4 +41,5 @@ public class Packages { referencedPackages.removeAll(definedPackages); return new PackageMetaData(definedPackages, referencedPackages); } + } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java index 4502f88d158..1bb6cb8976e 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java @@ -7,6 +7,7 @@ import com.yahoo.container.plugin.classanalysis.sampleclasses.Derived; import com.yahoo.container.plugin.classanalysis.sampleclasses.Dummy; import com.yahoo.container.plugin.classanalysis.sampleclasses.Fields; import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface1; +import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface3; import com.yahoo.container.plugin.classanalysis.sampleclasses.Methods; import org.junit.Test; @@ -24,6 +25,7 @@ import static org.junit.Assert.assertTrue; * @author Tony Vaagenes */ public class AnalyzeMethodBodyTest { + @Test public void require_that_class_of_locals_are_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Base.class))); @@ -65,6 +67,11 @@ public class AnalyzeMethodBodyTest { } @Test + public void require_that_functional_interface_usage_is_included() { + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Interface3.class))); + } + + @Test public void require_that_class_owning_method_handler_is_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(ClassWithMethod.class))); } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java index 9eacc0625b5..c2567e96ed9 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java @@ -8,6 +8,7 @@ import java.io.File; * @author ollivir */ public class TestUtilities { + public static ClassFileMetaData analyzeClass(Class<?> clazz) { return Analyze.analyzeClass(classFile(name(clazz))); } @@ -19,4 +20,5 @@ public class TestUtilities { public static String name(Class<?> clazz) { return clazz.getName(); } + } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Interface3.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Interface3.java new file mode 100644 index 00000000000..d492cd6ed50 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Interface3.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis.sampleclasses; + +/** + * Input for class analysis tests. + * + * @author bratseth + */ +public interface Interface3 extends java.util.function.Supplier<String> { + +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java index fcc7057dac8..3fba4b3381b 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/Methods.java @@ -3,6 +3,7 @@ package com.yahoo.container.plugin.classanalysis.sampleclasses; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Input for class analysis tests. @@ -10,6 +11,7 @@ import java.util.Map; */ @SuppressWarnings("unused") public class Methods { + public void method1() { Base b = new Base(); System.out.println(Fields.field2.size()); @@ -29,5 +31,15 @@ public class Methods { Derived d = new Derived(); } + public void method3() { + var result = methodTakingFunctionalArgument((Interface3)() -> "hello"); + System.out.println(result); + } + + public String methodTakingFunctionalArgument(Supplier<String> function) { + return function.get(); + } + public void methodTakingGenericArgument(Map<String, List<Dummy>> map) {} + } diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go index 6649353df77..490527df224 100644 --- a/client/go/cmd/test_test.go +++ b/client/go/cmd/test_test.go @@ -5,9 +5,6 @@ package cmd import ( - "fmt" - "github.com/vespa-engine/vespa/client/go/util" - "github.com/vespa-engine/vespa/client/go/vespa" "io/ioutil" "net/http" "net/url" @@ -16,6 +13,9 @@ import ( "strings" "testing" + "github.com/vespa-engine/vespa/client/go/util" + "github.com/vespa-engine/vespa/client/go/vespa" + "github.com/stretchr/testify/assert" ) @@ -40,7 +40,6 @@ func TestSuite(t *testing.T) { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) - fmt.Println(outBytes) assert.Equal(t, string(expectedBytes), outBytes) assert.Equal(t, "", errBytes) } diff --git a/client/go/util/io.go b/client/go/util/io.go index 23bfec84879..68b50733006 100644 --- a/client/go/util/io.go +++ b/client/go/util/io.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "strings" ) @@ -53,7 +54,8 @@ func ReaderToJSON(reader io.Reader) string { // AtomicWriteFile atomically writes data to filename. func AtomicWriteFile(filename string, data []byte) error { - tmpFile, err := ioutil.TempFile("", "vespa") + dir := filepath.Dir(filename) + tmpFile, err := ioutil.TempFile(dir, "vespa") if err != nil { return err } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index 72b489e3ca9..39ec3e1c647 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -32,7 +32,7 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer public CompiledQueryProfileRegistry(QueryProfilesConfig config, Executor executor) { QueryProfileRegistry registry = QueryProfileConfigurer.createFromConfig(config); typeRegistry = registry.getTypeRegistry(); - int maxConcurrent = Math.max(1, (int)(Runtime.getRuntime().availableProcessors() * 0.20)); + int maxConcurrent = 1; // TODO hold this one after Concurrency issue has been found: Math.max(1, (int)(Runtime.getRuntime().availableProcessors() * 0.20)); BlockingQueue<CompiledQueryProfile> doneQ = new LinkedBlockingQueue<>(); int started = 0; int completed = 0; diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index f6273fdf723..312fe175270 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.search.test; import com.yahoo.component.chain.Chain; +import com.yahoo.data.JsonProducer; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detection; @@ -15,6 +16,7 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; @@ -40,6 +42,7 @@ import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.yolean.Exceptions; +import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; @@ -69,6 +72,17 @@ import static org.junit.Assert.fail; public class QueryTestCase { @Test + public void testIt() throws Exception { + JSONObject newroot = new JSONObject("{\"key\": 3}"); + var hit = new FastHit(); + hit.setField("data", (JsonProducer)s -> s.append(newroot)); + var field = hit.getField("data"); + if (field instanceof JsonProducer) { + System.out.println((((JsonProducer) field).toJson())); + } + } + + @Test public void testSimpleFunctionality() { Query q = new Query(QueryTestCase.httpEncode("/sdfsd.html?query=this is a simple query&aParameter")); assertEquals("this is a simple query", q.getModel().getQueryString()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java index ad5bf1a2962..3e7da831bc4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.collections.AbstractFilteringList; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import java.util.Collection; @@ -21,6 +23,16 @@ public class LoadBalancerList extends AbstractFilteringList<LoadBalancer, LoadBa return matching(lb -> lb.state() == state); } + /** Returns the subset of load balancers in given cluster */ + public LoadBalancerList application(ApplicationId application) { + return matching(lb -> lb.id().application().equals(application)); + } + + /** Returns the subset of load balancers in given cluster */ + public LoadBalancerList cluster(ClusterSpec.Id cluster) { + return matching(lb -> lb.id().cluster().equals(cluster)); + } + public static LoadBalancerList copyOf(Collection<LoadBalancer> loadBalancers) { return new LoadBalancerList(loadBalancers, false); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java index d2c1aab72e2..7cbb8ef2764 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java @@ -2,20 +2,9 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.NodeAcl; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; -import java.util.Comparator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; import java.util.function.Predicate; -import java.util.stream.Collectors; /** * The load balancers of this node repo. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 8c358301b85..3da0506f2e1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -45,8 +45,8 @@ class Activator { /** Activate required resources for application guarded by given lock */ public void activate(Collection<HostSpec> hosts, long generation, ApplicationTransaction transaction) { - activateNodes(hosts, generation, transaction); - activateLoadBalancers(hosts, transaction); + NodeList newActive = activateNodes(hosts, generation, transaction); + activateLoadBalancers(hosts, newActive, transaction); } /** @@ -62,8 +62,9 @@ class Activator { * @param generation the application config generation that is activated * @param transaction transaction with operations to commit together with any operations done within the repository, * while holding the node repository lock on this application + * @return the nodes that will be active when transaction is committed */ - private void activateNodes(Collection<HostSpec> hosts, long generation, ApplicationTransaction transaction) { + private NodeList activateNodes(Collection<HostSpec> hosts, long generation, ApplicationTransaction transaction) { Instant activationTime = nodeRepository.clock().instant(); // Use one timestamp for all activation changes ApplicationId application = transaction.application(); Set<String> hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); @@ -95,6 +96,7 @@ class Activator { oldActive.not().retired(), newActive.not().retired()); unreserveParentsOf(reserved); + return newActive; } private void deactivate(NodeList toDeactivate, ApplicationTransaction transaction) { @@ -149,8 +151,8 @@ class Activator { } /** Activate load balancers */ - private void activateLoadBalancers(Collection<HostSpec> hosts, ApplicationTransaction transaction) { - loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(allClustersOf(hosts), transaction)); + private void activateLoadBalancers(Collection<HostSpec> hosts, NodeList newActive, ApplicationTransaction transaction) { + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(allClustersOf(hosts), newActive, transaction)); } private static Set<ClusterSpec> allClustersOf(Collection<HostSpec> hosts) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 5ff78c53f8a..04f084dd079 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -100,11 +100,11 @@ public class LoadBalancerProvisioner { * * Calling this when no load balancer has been prepared for given cluster is a no-op. */ - public void activate(Set<ClusterSpec> clusters, ApplicationTransaction transaction) { + public void activate(Set<ClusterSpec> clusters, NodeList newActive, ApplicationTransaction transaction) { Set<ClusterSpec.Id> activatingClusters = clusters.stream() .map(LoadBalancerProvisioner::effectiveId) .collect(Collectors.toSet()); - for (var cluster : loadBalancedClustersOf(transaction.application()).entrySet()) { + for (var cluster : loadBalancedClustersOf(newActive).entrySet()) { if (!activatingClusters.contains(cluster.getKey())) continue; Node clusterNode = cluster.getValue().first().get(); @@ -232,12 +232,13 @@ public class LoadBalancerProvisioner { /** Returns the nodes allocated to the given load balanced cluster */ private NodeList nodesOf(ClusterSpec.Id loadBalancedCluster, ApplicationId application) { - return loadBalancedClustersOf(application).getOrDefault(loadBalancedCluster, NodeList.copyOf(List.of())); + NodeList nodes = nodeRepository.nodes().list(Node.State.reserved, Node.State.active) + .owner(application); + return loadBalancedClustersOf(nodes).getOrDefault(loadBalancedCluster, NodeList.of()); } /** Returns the load balanced clusters of given application and their nodes */ - private Map<ClusterSpec.Id, NodeList> loadBalancedClustersOf(ApplicationId application) { - NodeList nodes = nodeRepository.nodes().list(Node.State.reserved, Node.State.active).owner(application); + private Map<ClusterSpec.Id, NodeList> loadBalancedClustersOf(NodeList nodes) { if (nodes.stream().anyMatch(node -> node.type() == NodeType.config)) { nodes = nodes.nodeType(NodeType.config).type(ClusterSpec.Type.admin); } else if (nodes.stream().anyMatch(node -> node.type() == NodeType.controller)) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 885efed9241..b1ee156f8e8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -12,24 +12,22 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.applicationmodel.InfrastructureApplication; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import org.junit.Test; -import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.SortedSet; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -47,6 +45,7 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); + private final NodeResources nodeResources = new NodeResources(1, 4, 10, 0.3); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final ProvisioningTester tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); @@ -55,22 +54,24 @@ public class LoadBalancerProvisionerTest { public void provision_load_balancer() { Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().list(app1).asList(); Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().list(app2).asList(); - ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); + ClusterSpec.Id containerCluster = ClusterSpec.Id.from("qrs1"); ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); // Provision a load balancer for each application var nodes = prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster)); assertEquals(1, lbApp1.get().size()); assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbApp1.get().get(0).instance().get().reals().size()); tester.activate(app1, nodes); - tester.activate(app2, prepare(app2, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")))); + tester.activate(app2, prepare(app2, clusterRequest(ClusterSpec.Type.container, containerCluster))); assertEquals(1, lbApp2.get().size()); + assertReals(app1, containerCluster, Node.State.active); + assertReals(app2, containerCluster, Node.State.active); // Reals are configured after activation assertEquals(app1, lbApp1.get().get(0).id().application()); - assertEquals(containerCluster1, lbApp1.get().get(0).id().cluster()); + assertEquals(containerCluster, lbApp1.get().get(0).id().cluster()); assertEquals(Collections.singleton(4443), lbApp1.get().get(0).instance().get().ports()); assertEquals("127.0.0.1", get(lbApp1.get().get(0).instance().get().reals(), 0).ipAddress()); assertEquals(4443, get(lbApp1.get().get(0).instance().get().reals(), 0).port()); @@ -84,7 +85,7 @@ public class LoadBalancerProvisionerTest { // Redeploying replaces failed node and removes it from load balancer tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().list(app1).asList().get(0); assertEquals(2, loadBalancer.instance().get().reals().size()); @@ -99,31 +100,15 @@ public class LoadBalancerProvisionerTest { // Add another container cluster to first app ClusterSpec.Id containerCluster2 = ClusterSpec.Id.from("qrs2"); tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.container, containerCluster2), clusterRequest(ClusterSpec.Type.content, contentCluster))); // Load balancer is provisioned for second container cluster - assertEquals(2, lbApp1.get().size()); - List<HostName> activeContainers = tester.getNodes(app1, Node.State.active) - .container().asList() - .stream() - .map(Node::hostname) - .map(HostName::from) - .sorted() - .collect(Collectors.toList()); - List<HostName> reals = lbApp1.get().stream() - .map(LoadBalancer::instance) - .flatMap(Optional::stream) - .map(LoadBalancerInstance::reals) - .flatMap(Collection::stream) - .map(Real::hostname) - .sorted() - .collect(Collectors.toList()); - assertEquals(activeContainers, reals); + assertReals(app1, containerCluster2, Node.State.active); // Cluster removal deactivates relevant load balancer - tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1))); + tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster))); assertEquals(2, lbApp1.get().size()); assertEquals("Deactivated load balancer for cluster " + containerCluster2, LoadBalancer.State.inactive, lbApp1.get().stream() @@ -131,9 +116,9 @@ public class LoadBalancerProvisionerTest { .map(LoadBalancer::state) .findFirst() .get()); - assertEquals("Load balancer for cluster " + containerCluster1 + " remains active", LoadBalancer.State.active, + assertEquals("Load balancer for cluster " + containerCluster + " remains active", LoadBalancer.State.active, lbApp1.get().stream() - .filter(lb -> lb.id().cluster().equals(containerCluster1)) + .filter(lb -> lb.id().cluster().equals(containerCluster)) .map(LoadBalancer::state) .findFirst() .get()); @@ -148,11 +133,11 @@ public class LoadBalancerProvisionerTest { // Application is redeployed with one cluster and load balancer is re-activated tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); - assertSame("Re-activated load balancer for " + containerCluster1, LoadBalancer.State.active, + assertSame("Re-activated load balancer for " + containerCluster, LoadBalancer.State.active, lbApp1.get().stream() - .filter(lb -> lb.id().cluster().equals(containerCluster1)) + .filter(lb -> lb.id().cluster().equals(containerCluster)) .map(LoadBalancer::state) .findFirst() .orElseThrow()); @@ -160,14 +145,14 @@ public class LoadBalancerProvisionerTest { // Next redeploy does not create a new load balancer instance because reals are unchanged tester.loadBalancerService().throwOnCreate(true); tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); // Routing is disabled through feature flag. Reals are removed on next deployment tester.loadBalancerService().throwOnCreate(false); flagSource.withBooleanFlag(PermanentFlags.DEACTIVATE_ROUTING.id(), true); tester.activate(app1, prepare(app1, - clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); List<LoadBalancer> activeLoadBalancers = lbApp1.get().stream() .filter(lb -> lb.state() == LoadBalancer.State.active) @@ -235,29 +220,12 @@ public class LoadBalancerProvisionerTest { @Test public void provision_load_balancer_config_server_cluster() { - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(InfrastructureApplication.CONFIG_SERVER.id()).asList(); - var cluster = ClusterSpec.Id.from("zone-config-servers"); - var nodes = prepare(InfrastructureApplication.CONFIG_SERVER.id(), Capacity.fromRequiredNodeType(NodeType.config), - clusterRequest(ClusterSpec.Type.admin, cluster)); - assertEquals(1, lbs.get().size()); - assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().get().reals().size()); - tester.activate(InfrastructureApplication.CONFIG_SERVER.id(), nodes); - assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); - assertEquals(cluster, lbs.get().get(0).id().cluster()); + provisionInfrastructureLoadBalancer(infraApp1, NodeType.config); } @Test public void provision_load_balancer_controller_cluster() { - ApplicationId controllerApp = ApplicationId.from("hosted-vespa", "controller", "default"); - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(controllerApp).asList(); - var cluster = ClusterSpec.Id.from("zone-config-servers"); - var nodes = prepare(controllerApp, Capacity.fromRequiredNodeType(NodeType.controller), - clusterRequest(ClusterSpec.Type.container, cluster)); - assertEquals(1, lbs.get().size()); - assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().get().reals().size()); - tester.activate(controllerApp, nodes); - assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); - assertEquals(cluster, lbs.get().get(0).id().cluster()); + provisionInfrastructureLoadBalancer(infraApp1, NodeType.controller); } @Test @@ -320,16 +288,69 @@ public class LoadBalancerProvisionerTest { assertTrue("No load balancer provisioned", tester.nodeRepository().loadBalancers().list(app1).asList().isEmpty()); } + @Test + public void load_balancer_targets_newly_active_nodes() { + ClusterSpec.Id container1 = ClusterSpec.Id.from("c1"); + // Initial deployment + { + Capacity capacity1 = Capacity.from(new ClusterResources(3, 1, nodeResources)); + Set<HostSpec> preparedHosts = prepare(app1, capacity1, clusterRequest(ClusterSpec.Type.container, container1)); + tester.activate(app1, preparedHosts); + } + assertReals(app1, container1, Node.State.active); + + // Next deployment removes a node + { + Capacity capacity1 = Capacity.from(new ClusterResources(2, 1, nodeResources)); + Set<HostSpec> preparedHosts = prepare(app1, capacity1, clusterRequest(ClusterSpec.Type.container, container1)); + tester.activate(app1, preparedHosts); + } + assertReals(app1, container1, Node.State.active); + } + + private void assertReals(ApplicationId application, ClusterSpec.Id cluster, Node.State... states) { + List<LoadBalancer> loadBalancers = tester.nodeRepository().loadBalancers().list(application).cluster(cluster).asList(); + assertEquals(1, loadBalancers.size()); + List<String> reals = loadBalancers.get(0).instance().get().reals().stream() + .map(real -> real.hostname().value()) + .sorted() + .collect(Collectors.toList()); + List<String> activeNodes = tester.nodeRepository().nodes().list(states) + .owner(application) + .cluster(cluster) + .hostnames().stream() + .sorted() + .collect(Collectors.toList()); + assertEquals("Load balancer targets active nodes of " + application + " in " + cluster, + activeNodes, reals); + } + + private void provisionInfrastructureLoadBalancer(ApplicationId application, NodeType nodeType) { + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(application).asList(); + var cluster = ClusterSpec.Id.from("infra-cluster"); + ClusterSpec.Type clusterType = nodeType == NodeType.config ? ClusterSpec.Type.admin : ClusterSpec.Type.container; + var nodes = prepare(application, Capacity.fromRequiredNodeType(nodeType), clusterRequest(clusterType, cluster)); + assertEquals(1, lbs.get().size()); + assertEquals("Prepare provisions load balancer with reserved nodes", 3, lbs.get().get(0).instance().get().reals().size()); + tester.activate(application, nodes); + assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); + assertEquals(cluster, lbs.get().get(0).id().cluster()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().nodes().deallocate(tester.nodeRepository().nodes().list().owner(application).asList(), Agent.system, this.getClass().getSimpleName()); } private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) { - return prepare(application, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true), specs); + return prepare(application, Capacity.from(new ClusterResources(2, 1, nodeResources), false, true), specs); } private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, ClusterSpec... specs) { - tester.makeReadyNodes(specs.length * 2, new NodeResources(1, 4, 10, 0.3), capacity.type()); + int nodeCount = capacity.minResources().nodes(); + if (capacity.type().isConfigServerLike()) { + nodeCount = 3; + } + tester.makeReadyNodes(specs.length * nodeCount, nodeResources, capacity.type()); Set<HostSpec> allNodes = new LinkedHashSet<>(); for (ClusterSpec spec : specs) { allNodes.addAll(tester.prepare(application, spec, capacity)); @@ -354,6 +375,9 @@ public class LoadBalancerProvisionerTest { } private static <T> T get(Set<T> set, int position) { + if (!(set instanceof SortedSet)) { + throw new IllegalArgumentException(set + " is not a sorted set"); + } return Iterators.get(set.iterator(), position, null); } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 1ba069818ba..7ebc3759813 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/features/attributefeature.h> #include <vespa/searchlib/features/closenessfeature.h> #include <vespa/searchlib/features/distancefeature.h> +#include <vespa/searchlib/features/great_circle_distance_feature.h> #include <vespa/searchlib/features/dotproductfeature.h> #include <vespa/searchlib/features/fieldlengthfeature.h> #include <vespa/searchlib/features/fieldmatchfeature.h> @@ -93,6 +94,7 @@ Test::Main() TEST_DO(testAttributeMatch()); TEST_FLUSH(); TEST_DO(testCloseness()); TEST_FLUSH(); TEST_DO(testMatchCount()); TEST_FLUSH(); + TEST_DO(testGreatCircleDistance()); TEST_FLUSH(); TEST_DO(testDistance()); TEST_FLUSH(); TEST_DO(testDistanceToPath()); TEST_FLUSH(); TEST_DO(testDotProduct()); TEST_FLUSH(); @@ -819,6 +821,67 @@ Test::assertFreshness(feature_t expFreshness, const vespalib::string & attr, uin ASSERT_TRUE(ft.execute(RankResult().addScore(feature, expFreshness).setEpsilon(EPS))); } +namespace { + +struct AirPort { + const char *tla; + double lat; + double lng; +}; + +std::pair<int32_t, int32_t> toXY(const AirPort &p) { + return std::make_pair((int)(p.lng * 1.0e6), + (int)(p.lat * 1.0e6)); +} + +GeoLocation toGL(const AirPort &p) { + int32_t x = (int)(p.lng * 1.0e6); + int32_t y = (int)(p.lat * 1.0e6); + GeoLocation::Point gp{x, y}; + return GeoLocation{gp}; +} + +} + +void +Test::testGreatCircleDistance() +{ + { // Test blueprint. + GreatCircleDistanceBlueprint pt; + EXPECT_TRUE(assertCreateInstance(pt, "great_circle_distance")); + StringList params, in, out; + FT_SETUP_FAIL(pt, params); + FtIndexEnvironment idx_env; + idx_env + .getBuilder() + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos_zcurve"); + FT_SETUP_OK(pt, idx_env, params.add("pos"), in, + out.add("km").add("latitude").add("longitude")); + FT_DUMP_EMPTY(_factory, "great_circle_distance"); + } + { // Test executor. + FtFeatureTest ft(_factory, "great_circle_distance(pos)"); + const AirPort SFO = { "SFO", 37.618806, -122.375416 }; + const AirPort TRD = { "TRD", 63.457556, 10.924250 }; + std::vector<std::pair<int32_t,int32_t>> pos = { toXY(SFO), toXY(TRD) }; + setupForDistanceTest(ft, "pos_zcurve", pos, true); + const AirPort LHR = { "LHR", 51.477500, -0.461388 }; + const AirPort JFK = { "JFK", 40.639928, -73.778692 }; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", toGL(LHR)}); + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", toGL(JFK)}); + ASSERT_TRUE(ft.setup()); + double exp = 1494; // according to gcmap.com + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(10.0). + addScore("great_circle_distance(pos)", exp))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(10.0). + addScore("great_circle_distance(pos).km", exp))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-9). + addScore("great_circle_distance(pos).latitude", TRD.lat))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-9). + addScore("great_circle_distance(pos).longitude", TRD.lng))); + } +} + void Test::testDistance() { @@ -830,7 +893,7 @@ Test::testDistance() StringList params, in, out; FT_SETUP_FAIL(pt, params); FT_SETUP_OK(pt, params.add("pos"), in, - out.add("out").add("index").add("latitude").add("longitude")); + out.add("out").add("index").add("latitude").add("longitude").add("km")); FT_DUMP_EMPTY(_factory, "distance"); } @@ -963,6 +1026,8 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions, ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-4). addScore("distance(pos)", exp))); + ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-4). + addScore("distance(pos).km", exp * 0.00011119508023))); ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-30). addScore("distance(pos).index", hit_index))); ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-9). diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index 58e6b4953cc..ad21d7d7ccc 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -19,6 +19,7 @@ public: void testAttributeMatch(); void testCloseness(); void testMatchCount(); + void testGreatCircleDistance(); void testDistance(); void testDistanceToPath(); void testDotProduct(); diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 9d4119a7faa..88531a46cb1 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(searchlib_features OBJECT debug_wait.cpp dense_tensor_attribute_executor.cpp direct_tensor_attribute_executor.cpp + great_circle_distance_feature.cpp distancefeature.cpp distancetopathfeature.cpp documenttestutils.cpp diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 518ade2a8f5..57fa5fc7cee 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -135,10 +135,15 @@ DistanceExecutor::DistanceExecutor(GeoLocationSpecPtrs locations, void DistanceExecutor::execute(uint32_t docId) { - outputs().set_number(0, calculateDistance(docId)); + static constexpr double earth_mean_radius = 6371.0088; + static constexpr double deg_to_rad = M_PI / 180.0; + static constexpr double km_from_internal = 1.0e-6 * deg_to_rad * earth_mean_radius; + feature_t internal_d = calculateDistance(docId); + outputs().set_number(0, internal_d); outputs().set_number(1, _best_index); outputs().set_number(2, _best_y * 1.0e-6); // latitude outputs().set_number(3, _best_x * 1.0e-6); // longitude + outputs().set_number(4, internal_d * km_from_internal); // km } const feature_t DistanceExecutor::DEFAULT_DISTANCE(6400000000.0); @@ -178,6 +183,7 @@ DistanceBlueprint::setup_geopos(const IIndexEnvironment & env, describeOutput("index", "Index in array of closest point"); describeOutput("latitude", "Latitude of closest point"); describeOutput("longitude", "Longitude of closest point"); + describeOutput("km", "Distance in kilometer units"); env.hintAttributeAccess(_arg_string); return true; } diff --git a/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.cpp b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.cpp new file mode 100644 index 00000000000..eb47c88ecd0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.cpp @@ -0,0 +1,190 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "great_circle_distance_feature.h" +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchlib/common/geo_location_spec.h> +#include <vespa/searchlib/fef/matchdata.h> +#include <vespa/document/datatype/positiondatatype.h> +#include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/util/stash.h> +#include <cmath> +#include <limits> +#include "utils.h" + +#include <vespa/log/log.h> +LOG_SETUP(".features.great_circle_distance_feature"); + +using namespace search::fef; +using namespace search::index::schema; +using vespalib::Issue; + +namespace search::features { + +feature_t GCDExecutor::calculateGCD(uint32_t docId) { + feature_t dist = std::numeric_limits<feature_t>::max(); + if (_locations.empty()) { + return dist; + } + _intBuf.fill(*_pos, docId); + uint32_t numValues = _intBuf.size(); + int32_t docx = 0; + int32_t docy = 0; + for (auto loc : _locations) { + for (uint32_t i = 0; i < numValues; ++i) { + vespalib::geo::ZCurve::decode(_intBuf[i], &docx, &docy); + double lat = docy / 1.0e6; + double lng = docx / 1.0e6; + double d = loc.km_great_circle_distance(lat, lng); + if (d < dist) { + dist = d; + _best_lat = lat; + _best_lng = lng; + } + } + } + return dist; +} + +GCDExecutor::GCDExecutor(GeoLocationSpecPtrs locations, const attribute::IAttributeVector * pos) + : FeatureExecutor(), + _locations(), + _pos(pos), + _intBuf() +{ + if (_pos == nullptr) { + return; + } + _intBuf.allocate(_pos->getMaxValueCount()); + for (const auto * p : locations) { + if (p && p->location.valid()) { + double lat = p->location.point.y * 1.0e-6; + double lng = p->location.point.x * 1.0e-6; + _locations.emplace_back(search::common::GeoGcd{lat, lng}); + } + } +} + +void +GCDExecutor::execute(uint32_t docId) +{ + outputs().set_number(0, calculateGCD(docId)); + outputs().set_number(1, _best_lat); // latitude + outputs().set_number(2, _best_lng); // longitude +} + + +GreatCircleDistanceBlueprint::GreatCircleDistanceBlueprint() : + Blueprint("great_circle_distance"), + _attr_name() +{ +} + +GreatCircleDistanceBlueprint::~GreatCircleDistanceBlueprint() = default; + +void GreatCircleDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const +{ +} + +Blueprint::UP +GreatCircleDistanceBlueprint::createInstance() const +{ + return std::make_unique<GreatCircleDistanceBlueprint>(); +} + +bool +GreatCircleDistanceBlueprint::setup_geopos(const IIndexEnvironment & env, const vespalib::string &attr) +{ + _attr_name = attr; + describeOutput("km", "The distance (in km) from the query position."); + describeOutput("latitude", "Latitude of closest point"); + describeOutput("longitude", "Longitude of closest point"); + env.hintAttributeAccess(_attr_name); + return true; +} + + +bool +GreatCircleDistanceBlueprint::setup(const IIndexEnvironment & env, + const ParameterList & params) +{ + vespalib::string arg = params[0].getValue(); + if (params.size() == 1) { + // params[0] = attribute name + } else if (params.size() == 2) { + // params[0] = "field" + // params[1] = attribute name + if (arg == "field") { + arg = params[1].getValue(); + } else { + LOG(error, "first argument must be 'field' but was '%s'", arg.c_str()); + return false; + } + } else { + LOG(error, "bad params.size() = %zd", params.size()); + return false; + } + vespalib::string z = document::PositionDataType::getZCurveFieldName(arg); + const auto *fi = env.getFieldByName(z); + if (fi != nullptr && fi->hasAttribute()) { + auto dt = fi->get_data_type(); + auto ct = fi->collection(); + if (dt == DataType::INT64) { + if (ct == CollectionType::SINGLE || ct == CollectionType::ARRAY) { + return setup_geopos(env, z); + } + } + } + if (env.getFieldByName(arg) == nullptr && fi == nullptr) { + LOG(error, "unknown field '%s' for rank feature %s\n", arg.c_str(), getName().c_str()); + } else { + LOG(error, "field '%s' must be type position and attribute for rank feature %s\n", arg.c_str(), getName().c_str()); + } + return false; +} + +FeatureExecutor & +GreatCircleDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const +{ + // expect geo pos: + const search::attribute::IAttributeVector * pos = nullptr; + GeoLocationSpecPtrs matching_locs; + GeoLocationSpecPtrs other_locs; + + for (auto loc_ptr : env.getAllLocations()) { + if (loc_ptr && loc_ptr->location.valid()) { + if (loc_ptr->field_name == _attr_name) { + LOG(debug, "found loc from query env matching '%s'", _attr_name.c_str()); + matching_locs.push_back(loc_ptr); + } else { + LOG(debug, "found loc(%s) from query env not matching arg(%s)", + loc_ptr->field_name.c_str(), _attr_name.c_str()); + other_locs.push_back(loc_ptr); + } + } + } + if (matching_locs.empty() && other_locs.empty()) { + LOG(debug, "createExecutor: no valid locations"); + return stash.create<GCDExecutor>(matching_locs, nullptr); + } + LOG(debug, "createExecutor: valid location, attribute='%s'", _attr_name.c_str()); + pos = env.getAttributeContext().getAttribute(_attr_name); + if (pos != nullptr) { + if (!pos->isIntegerType()) { + Issue::report("distance feature: The position attribute '%s' is not an integer attribute.", + pos->getName().c_str()); + pos = nullptr; + } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { + Issue::report("distance feature: The position attribute '%s' is a weighted set attribute.", + pos->getName().c_str()); + pos = nullptr; + } + } else { + Issue::report("distance feature: The position attribute '%s' was not found.", _attr_name.c_str()); + } + LOG(debug, "use '%s' locations with pos=%p", matching_locs.empty() ? "other" : "matching", pos); + return stash.create<GCDExecutor>(matching_locs.empty() ? other_locs : matching_locs, pos); +} + +} diff --git a/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.h b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.h new file mode 100644 index 00000000000..d44e4f5569b --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/great_circle_distance_feature.h @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchlib/common/geo_gcd.h> + +namespace search::features { + +/** Convenience typedef. */ +using GeoLocationSpecPtrs = std::vector<const search::common::GeoLocationSpec *>; + +/** + * Implements the executor for the great circle distance feature. + */ +class GCDExecutor : public fef::FeatureExecutor { +private: + std::vector<search::common::GeoGcd> _locations; + const attribute::IAttributeVector * _pos; + attribute::IntegerContent _intBuf; + feature_t _best_lat; + feature_t _best_lng; + + feature_t calculateGCD(uint32_t docId); +public: + /** + * Constructs an executor for the GCD feature. + * + * @param locations location objects associated with the query environment. + * @param pos the attribute to use for positions (expects zcurve encoding). + */ + GCDExecutor(GeoLocationSpecPtrs locations, const attribute::IAttributeVector * pos); + void execute(uint32_t docId) override; +}; + +/** + * Implements the blueprint for the GCD executor. + */ +class GreatCircleDistanceBlueprint : public fef::Blueprint { +private: + vespalib::string _attr_name; + bool setup_geopos(const fef::IIndexEnvironment & env, const vespalib::string &attr); +public: + GreatCircleDistanceBlueprint(); + ~GreatCircleDistanceBlueprint(); + void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; + fef::Blueprint::UP createInstance() const override; + fef::ParameterDescriptions getDescriptions() const override { + return fef::ParameterDescriptions().desc().string().desc().string().string(); + } + bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index f2d5bd745ac..2bc8a349d1b 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -10,6 +10,7 @@ #include "debug_attribute_wait.h" #include "debug_wait.h" #include "distancefeature.h" +#include "great_circle_distance_feature.h" #include "distancetopathfeature.h" #include "dotproductfeature.h" #include "element_completeness_feature.h" @@ -126,6 +127,7 @@ void setup_search_features(fef::IBlueprintRegistry & registry) registry.addPrototype(std::make_shared<GlobalSequenceBlueprint>()); registry.addPrototype(std::make_shared<OnnxBlueprint>("onnx")); registry.addPrototype(std::make_shared<OnnxBlueprint>("onnxModel")); + registry.addPrototype(std::make_shared<GreatCircleDistanceBlueprint>()); // Ranking Expression auto replacers = std::make_unique<ListExpressionReplacer>(); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java index 1874bd42e16..b51210d22ea 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java @@ -16,6 +16,10 @@ import org.apache.hc.core5.net.URIAuthority; import org.apache.hc.core5.reactor.IOReactorConfig; import org.apache.hc.core5.util.Timeout; +import org.apache.hc.core5.function.Factory; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import javax.net.ssl.SSLEngine; + import javax.net.ssl.SSLContext; import java.io.IOException; import java.net.URI; @@ -132,6 +136,8 @@ class ApacheCluster implements Cluster { throw new IllegalStateException("No adequate SSL cipher suites supported by the JVM"); ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create() + .setTlsDetailsFactory(sslEngine -> + new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol())) .setCiphers(allowedCiphers) .setSslContext(sslContext); if (builder.hostnameVerifier != null) |