Add host indices to duper model

5 files changed, 157 insertions, 50 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index aa0b9e81c44..9bfba36a2d3 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -2,16 +2,20 @@
 package com.yahoo.vespa.service.duper;
 
 import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.service.monitor.DuperModelListener;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.TreeMap;
+import java.util.Set;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * A non-thread-safe mutable container of ApplicationInfo, also taking care of listeners on changes.
@@ -21,41 +25,83 @@ import java.util.logging.Logger;
 public class DuperModel {
     private static Logger logger = Logger.getLogger(DuperModel.class.getName());
 
-    private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>();
+    private final Map<ApplicationId, ApplicationInfo> applicationsById = new HashMap<>();
+    private final Map<HostName, ApplicationInfo> applicationsByHostname = new HashMap<>();
+    private final Map<ApplicationId, Set<HostName>> hostnamesById = new HashMap<>();
+
     private final List<DuperModelListener> listeners = new ArrayList<>();
     private boolean isComplete = false;
 
     public void registerListener(DuperModelListener listener) {
-        applications.values().forEach(listener::applicationActivated);
+        applicationsById.values().forEach(listener::applicationActivated);
         listeners.add(listener);
     }
 
-    public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; }
+    void setComplete() { this.isComplete = true; }
     public boolean isComplete() { return isComplete; }
 
+    public int numberOfApplications() {
+        return applicationsById.size();
+    }
+
+    public int numberOfHosts() {
+        return applicationsByHostname.size();
+    }
+
     public boolean contains(ApplicationId applicationId) {
-        return applications.containsKey(applicationId);
+        return applicationsById.containsKey(applicationId);
+    }
+
+    public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) {
+        return Optional.ofNullable(applicationsById.get(applicationId));
+    }
+
+    public Optional<ApplicationInfo> getApplicationInfo(HostName hostName) {
+        return Optional.ofNullable(applicationsByHostname.get(hostName));
+    }
+
+    public List<ApplicationInfo> getApplicationInfos() {
+        return List.copyOf(applicationsById.values());
     }
 
     public void add(ApplicationInfo applicationInfo) {
-        applications.put(applicationInfo.getApplicationId(), applicationInfo);
-        logger.log(LogLevel.DEBUG, "Added " + applicationInfo.getApplicationId());
+        ApplicationInfo oldApplicationInfo = applicationsById.put(applicationInfo.getApplicationId(), applicationInfo);
+
+        final String logPrefix;
+        if (oldApplicationInfo == null) {
+            logPrefix = isComplete ? "New application " : "Bootstrapped application ";
+        } else {
+            logPrefix = isComplete ? "Reactivated application " : "Rebootstrapped application ";
+        }
+        logger.log(LogLevel.INFO, logPrefix + applicationInfo.getApplicationId());
+
+        Set<HostName> oldHostnames = hostnamesById.get(applicationInfo.getApplicationId());
+        if (oldHostnames != null) {
+            oldHostnames.forEach(applicationsByHostname::remove);
+        }
+
+        Set<HostName> hostnames = applicationInfo.getModel().getHosts().stream()
+                .map(HostInfo::getHostname)
+                .map(HostName::from)
+                .collect(Collectors.toSet());
+
+        hostnamesById.put(applicationInfo.getApplicationId(), hostnames);
+        hostnames.forEach(hostname -> applicationsByHostname.put(hostname, applicationInfo));
+
         listeners.forEach(listener -> listener.applicationActivated(applicationInfo));
     }
 
     public void remove(ApplicationId applicationId) {
-        if (applications.remove(applicationId) != null) {
-            logger.log(LogLevel.DEBUG, "Removed " + applicationId);
-            listeners.forEach(listener -> listener.applicationRemoved(applicationId));
+        Set<HostName> hostnames = hostnamesById.remove(applicationId);
+        if (hostnames != null) {
+            hostnames.forEach(applicationsByHostname::remove);
         }
-    }
 
-    public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) {
-        return Optional.ofNullable(applications.get(applicationId));
-    }
+        ApplicationInfo application = applicationsById.remove(applicationId);
 
-    public List<ApplicationInfo> getApplicationInfos() {
-        logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size());
-        return List.copyOf(applications.values());
+        if (application != null || hostnames != null) {
+            logger.log(LogLevel.INFO, "Removed application " + applicationId);
+            listeners.forEach(listener -> listener.applicationRemoved(applicationId));
+        }
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
index 9c819d89fb2..9c93bc1d390 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
@@ -179,6 +179,12 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi
         }
     }
 
+    public Optional<ApplicationInfo> getApplicationInfo(HostName hostname) {
+        synchronized (monitor) {
+            return duperModel.getApplicationInfo(hostname);
+        }
+    }
+
     public List<ApplicationInfo> getApplicationInfos() {
         synchronized (monitor) {
             return duperModel.getApplicationInfos();
@@ -187,7 +193,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi
 
     private void maybeSetDuperModelAsComplete() {
         if (superModelIsComplete && infraApplicationsIsComplete) {
-            duperModel.setCompleteness(true);
+            duperModel.setComplete();
         }
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java
index 45a9d1c77c5..86bb892b162 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -50,40 +49,25 @@ public class ModelGenerator {
                 .collect(Collectors.toSet());
     }
 
-    public Optional<ApplicationInstance> toApplication(List<ApplicationInfo> applicationInfos,
-                                                       HostName hostname,
-                                                       ServiceStatusProvider serviceStatusProvider) {
-        for (var applicationInfo : applicationInfos) {
-            var generator = new ApplicationInstanceGenerator(applicationInfo, zone);
-            if (generator.containsHostname(hostname)) {
-                return Optional.of(generator.makeApplicationInstance(serviceStatusProvider));
-            }
-        }
-
-        return Optional.empty();
-    }
-
     public ApplicationInstance toApplication(ApplicationInfo applicationInfo,
                                              ServiceStatusProvider serviceStatusProvider) {
         var generator = new ApplicationInstanceGenerator(applicationInfo, zone);
         return generator.makeApplicationInstance(serviceStatusProvider);
     }
 
-    public List<ServiceInstance> toServices(List<ApplicationInfo> applicationInfos,
+    public List<ServiceInstance> toServices(ApplicationInfo applicationInfo,
                                             HostName hostname,
                                             ServiceStatusProvider serviceStatusProvider) {
-        for (var applicationInfo : applicationInfos) {
-            var generator = new ApplicationInstanceGenerator(applicationInfo, zone);
-            ApplicationInstance applicationInstance = generator.makeApplicationInstanceLimitedTo(
-                    hostname, serviceStatusProvider);
+        var generator = new ApplicationInstanceGenerator(applicationInfo, zone);
+        ApplicationInstance applicationInstance = generator.makeApplicationInstanceLimitedTo(
+                hostname, serviceStatusProvider);
 
-            List<ServiceInstance> serviceInstances = applicationInstance.serviceClusters().stream()
-                    .flatMap(cluster -> cluster.serviceInstances().stream())
-                    .collect(Collectors.toList());
+        List<ServiceInstance> serviceInstances = applicationInstance.serviceClusters().stream()
+                .flatMap(cluster -> cluster.serviceInstances().stream())
+                .collect(Collectors.toList());
 
-            if (serviceInstances.size() > 0) {
-                return serviceInstances;
-            }
+        if (serviceInstances.size() > 0) {
+            return serviceInstances;
         }
 
         return List.of();
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java
index 641bc5ca7ff..d3d7c151a9b 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java
@@ -56,8 +56,13 @@ public class ServiceModelProvider implements ServiceMonitor {
 
     @Override
     public Optional<ApplicationInstance> getApplication(HostName hostname) {
-        // TODO(hakonhall): Lookup the applicationInfo from the hostname.
-        return modelGenerator.toApplication(applicationInfos(), hostname, serviceStatusProvider);
+        Optional<ApplicationInfo> applicationInfo =
+                duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname));
+        if (applicationInfo.isEmpty()) {
+            return Optional.empty();
+        }
+
+        return Optional.of(modelGenerator.toApplication(applicationInfo.get(), serviceStatusProvider));
     }
 
     @Override
@@ -68,8 +73,13 @@ public class ServiceModelProvider implements ServiceMonitor {
 
     @Override
     public List<ServiceInstance> getServiceInstancesOn(HostName hostname) {
-        // TODO(hakonhall): Lookup the applicationInfo from the hostname.
-        return modelGenerator.toServices(applicationInfos(), hostname, serviceStatusProvider);
+        Optional<ApplicationInfo> applicationInfo =
+                duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname));
+        if (applicationInfo.isEmpty()) {
+            return List.of();
+        }
+
+        return modelGenerator.toServices(applicationInfo.get(), hostname, serviceStatusProvider);
     }
 
     @Override
@@ -85,4 +95,9 @@ public class ServiceModelProvider implements ServiceMonitor {
     private List<ApplicationInfo> applicationInfos() {
         return duperModelManager.getApplicationInfos();
     }
+
+    /** The duper model uses HostName from config.provision, which is more natural than applicationmodel. */
+    private com.yahoo.config.provision.HostName toConfigProvisionHostName(HostName hostname) {
+        return com.yahoo.config.provision.HostName.from(hostname.s());
+    }
 }
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
index dc90035be71..0cc108bae9e 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
@@ -2,12 +2,18 @@
 package com.yahoo.vespa.service.duper;
 
 import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.Model;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.service.monitor.DuperModelListener;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -23,32 +29,55 @@ import static org.mockito.Mockito.when;
  */
 public class DuperModelTest {
     private final DuperModel duperModel = new DuperModel();
-    private final ApplicationInfo application1 = mock(ApplicationInfo.class);
+
     private final ApplicationId id1 = ApplicationId.fromSerializedForm("tenant:app1:default");
+    private final ApplicationInfo application1 = mock(ApplicationInfo.class);
+    private final HostName hostname1_1 = HostName.from("hostname1-1");
+    private final HostName hostname1_2 = HostName.from("hostname1-2");
+
     private final ApplicationId id2 = ApplicationId.fromSerializedForm("tenant:app2:default");
     private final ApplicationInfo application2 = mock(ApplicationInfo.class);
+    private final HostName hostname2_1 = HostName.from("hostname2-1");
+
     private final DuperModelListener listener1 = mock(DuperModelListener.class);
 
     @Before
     public void setUp() {
-        when(application1.getApplicationId()).thenReturn(id1);
-        when(application2.getApplicationId()).thenReturn(id2);
+        setUpApplication(id1, application1, hostname1_1, hostname1_2);
+        setUpApplication(id2, application2, hostname2_1);
+    }
+
+    private void setUpApplication(ApplicationId id, ApplicationInfo info, HostName... hostnames) {
+        when(info.getApplicationId()).thenReturn(id);
+
+        Model model = mock(Model.class);
+        when(info.getModel()).thenReturn(model);
+
+        List<HostInfo> hostInfos = Arrays.stream(hostnames)
+                .map(hostname -> new HostInfo(hostname.value(), List.of()))
+                .collect(Collectors.toList());
+        when(model.getHosts()).thenReturn(hostInfos);
     }
 
     @Test
     public void test() {
+        assertEquals(0, duperModel.numberOfApplications());
+
         duperModel.add(application1);
         assertTrue(duperModel.contains(id1));
         assertEquals(Arrays.asList(application1), duperModel.getApplicationInfos());
+        assertEquals(1, duperModel.numberOfApplications());
 
         duperModel.registerListener(listener1);
         verify(listener1, times(1)).applicationActivated(application1);
         verifyNoMoreInteractions(listener1);
 
         duperModel.remove(id2);
+        assertEquals(1, duperModel.numberOfApplications());
         verifyNoMoreInteractions(listener1);
 
         duperModel.add(application2);
+        assertEquals(2, duperModel.numberOfApplications());
         verify(listener1, times(1)).applicationActivated(application2);
         verifyNoMoreInteractions(listener1);
 
@@ -61,4 +90,31 @@ public class DuperModelTest {
         duperModel.remove(id1);
         verifyNoMoreInteractions(listener1);
     }
+
+    @Test
+    public void hostIndices() {
+        assertEquals(0, duperModel.numberOfHosts());
+
+        duperModel.add(application1);
+        assertEquals(2, duperModel.numberOfHosts());
+        assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1));
+
+        duperModel.add(application2);
+        assertEquals(3, duperModel.numberOfHosts());
+        assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1));
+
+        duperModel.remove(application1.getApplicationId());
+        assertEquals(1, duperModel.numberOfHosts());
+        assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1));
+
+        // Remove hostname2_1 and add hostname1_1 added to id2
+        setUpApplication(id2, application2, hostname1_1);
+        duperModel.add(application2);
+        assertEquals(1, duperModel.numberOfHosts());
+        assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1));
+    }
 }
\ No newline at end of file