Add RealDataScenarioTest

3 files changed, 141 insertions, 5 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
index 8e691b538a1..3cae4a5a5ea 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
@@ -4,14 +4,11 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationTransaction;
 import com.yahoo.config.provision.ClusterMembership;
-import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.ParentHostUnavailableException;
-import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.transaction.Mutex;
-import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
index 511b397be1a..39375494d01 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
@@ -137,9 +137,8 @@ class NodesResponse extends HttpResponse {
         if ( ! allFields) return;
         object.setString("id", node.hostname());
         object.setString("state", NodeSerializer.toString(node.state()));
-        object.setString("type", node.type().name());
-        object.setString("hostname", node.hostname());
         object.setString("type", NodeSerializer.toString(node.type()));
+        object.setString("hostname", node.hostname());
         if (node.parentHostname().isPresent()) {
             object.setString("parentHostname", node.parentHostname().get());
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java
new file mode 100644
index 00000000000..af14876c20c
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java
@@ -0,0 +1,140 @@
+package com.yahoo.vespa.hosted.provision;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.provision.ActivationContext;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationTransaction;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.Cloud;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.ProvisionLock;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.config.provisioning.FlavorsConfig;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.persistence.NodeSerializer;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import com.yahoo.vespa.model.builder.xml.dom.DomConfigPayloadBuilder;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.any;
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
+import static com.yahoo.config.provision.NodeResources.StorageType.local;
+import static com.yahoo.config.provision.NodeResources.StorageType.remote;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Scenario tester with real node-repository data loaded from ZK snapshot file
+ *
+ * @author valerijf
+ */
+public class RealDataScenarioTest {
+    private static final Logger log = Logger.getLogger(RealDataScenarioTest.class.getSimpleName());
+
+    @Ignore
+    @Test
+    public void test() {
+        ProvisioningTester tester = new ProvisioningTester.Builder()
+                .zone(new Zone(Cloud.builder().dynamicProvisioning(true).build(), SystemName.defaultSystem(), Environment.prod, RegionName.defaultName()))
+                .flavorsConfig(parseFlavors(Paths.get("flavors.xml")))
+                .build();
+        initFromZk(tester.nodeRepository(), Paths.get("snapshot"));
+
+        ApplicationId app = ApplicationId.from("tenant", "app", "default");
+        Version version = Version.fromString("7.123.4");
+
+        Capacity[] capacities = new Capacity[]{
+                Capacity.from(new ClusterResources(1, 1, new NodeResources(0.5, 4, 50, 0.3, any, remote))),
+                Capacity.from(new ClusterResources(4, 1, new NodeResources(8, 16, 100, 0.3, fast, remote))),
+                Capacity.from(new ClusterResources(2, 1, new NodeResources(4, 8, 100, 0.3, fast, local)))
+        };
+        ClusterSpec[] specs = new ClusterSpec[]{
+                ClusterSpec.request(ClusterSpec.Type.admin, ClusterSpec.Id.from("logserver")).vespaVersion(version).build(),
+                ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container")).vespaVersion(version).build(),
+                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content")).vespaVersion(version).build()
+        };
+
+        deploy(tester, app, specs, capacities);
+        tester.nodeRepository().list(app).cluster(specs[1].id()).forEach(System.out::println);
+    }
+
+    private void deploy(ProvisioningTester tester, ApplicationId app, ClusterSpec[] specs, Capacity[] capacities) {
+        List<HostSpec> hostSpecs = IntStream.range(0, capacities.length)
+                .mapToObj(i -> tester.provisioner().prepare(app, specs[i], capacities[i], log::log).stream())
+                .flatMap(s -> s)
+                .collect(Collectors.toList());
+        NestedTransaction transaction = new NestedTransaction();
+        tester.provisioner().activate(hostSpecs, new ActivationContext(0), new ApplicationTransaction(new ProvisionLock(app, () -> {}), transaction));
+        transaction.commit();
+    }
+
+    private static FlavorsConfig parseFlavors(Path path) {
+        try {
+            var element = XmlHelper.getDocumentBuilder().parse(path.toFile()).getDocumentElement();
+            return ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(element)).toInstance(FlavorsConfig.class, "");
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static void initFromZk(NodeRepository nodeRepository, Path pathToZkSnapshot) {
+        NodeSerializer nodeSerializer = new NodeSerializer(nodeRepository.flavors(), 1000);
+        AtomicReference<Node.State> state = new AtomicReference<>();
+        Pattern zkNodePathPattern = Pattern.compile(".?/provision/v1/([a-z]+)/[a-z0-9.-]+\\.(com|cloud).?");
+        Consumer<String> consumer = input -> {
+            if (state.get() != null) {
+                String json = input.substring(input.indexOf("{\""), input.lastIndexOf('}') + 1);
+                Node node = nodeSerializer.fromJson(state.get(), json.getBytes(UTF_8));
+                nodeRepository.database().addNodesInState(List.of(node), state.get(), Agent.system);
+                state.set(null);
+            } else {
+                Matcher matcher = zkNodePathPattern.matcher(input);
+                if (!matcher.matches()) return;
+                String stateStr = matcher.group(1);
+                Node.State s = "deallocated".equals(stateStr) ? Node.State.inactive :
+                        "allocated".equals(stateStr) ? Node.State.active : Node.State.valueOf(stateStr);
+                state.set(s);
+            }
+        };
+
+        try (BufferedReader reader = new BufferedReader(
+                new InputStreamReader(Files.newInputStream(pathToZkSnapshot), UTF_8))) {
+            StringBuilder sb = new StringBuilder(1000);
+            for (int r; (r = reader.read()) != -1; ) {
+                if (r < 0x20 || r >= 0x7F) {
+                    if (sb.length() > 0) {
+                        consumer.accept(sb.toString());
+                        sb.setLength(0);
+                    }
+                } else sb.append((char) r);
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}