Minor cleanup (logging, config generation, simplification)

1 files changed, 44 insertions, 60 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
index 90667e03540..3a88779a979 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
@@ -1,34 +1,34 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.application.validation;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.collections.Pair;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.InvalidLogFormatException;
 import com.yahoo.log.LogMessage;
 import com.yahoo.searchdefinition.DistributableResource;
 import com.yahoo.searchdefinition.OnnxModel;
 import com.yahoo.searchdefinition.RankExpressionBody;
-import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
-import com.yahoo.vespa.defaults.Defaults;
-import com.yahoo.yolean.Exceptions;
 import com.yahoo.system.ProcessExecuter;
 import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.search.AttributesConfig;
-import com.yahoo.collections.Pair;
-import com.yahoo.config.ConfigInstance;
 import com.yahoo.vespa.config.search.ImportedFieldsConfig;
 import com.yahoo.vespa.config.search.IndexschemaConfig;
 import com.yahoo.vespa.config.search.RankProfilesConfig;
-import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
-import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.search.AbstractSearchCluster;
 import com.yahoo.vespa.model.search.DocumentDatabase;
 import com.yahoo.vespa.model.search.IndexedSearchCluster;
 import com.yahoo.vespa.model.search.SearchCluster;
+import com.yahoo.yolean.Exceptions;
 
 import java.io.File;
 import java.io.IOException;
@@ -36,13 +36,13 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.logging.Logger;
-import java.util.logging.Level;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
- * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done
+ * Validates rank setup for all content clusters (rank-profiles, index-schema, attributes configs), validation is done
  * by running the binary 'vespa-verify-ranksetup-bin'
  *
  * @author vegardh
@@ -69,17 +69,17 @@ public class RankSetupValidator extends Validator {
 
             for (AbstractSearchCluster cluster : model.getSearchClusters()) {
                 // Skipping rank expression checking for streaming clusters, not implemented yet
-                if (cluster.isRealtime()) {
-                    IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
-                    String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/";
-                    for (DocumentDatabase docDb : sc.getDocumentDbs()) {
-                        final String name = docDb.getDerivedConfiguration().getSearch().getName();
-                        String searchDir = clusterDir + name + "/";
-                        writeConfigs(searchDir, docDb);
-                        writeExtraVerifyRanksetupConfig(searchDir, docDb);
-                        if ( ! validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) {
-                            return;
-                        }
+                if (cluster.isStreaming()) continue;
+
+                IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
+                String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/";
+                for (DocumentDatabase docDb : sc.getDocumentDbs()) {
+                    final String name = docDb.getDerivedConfiguration().getSearch().getName();
+                    String searchDir = clusterDir + name + "/";
+                    writeConfigs(searchDir, docDb);
+                    writeExtraVerifyRanksetupConfig(searchDir, docDb);
+                    if (!validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) {
+                        return;
                     }
                 }
             }
@@ -95,16 +95,13 @@ public class RankSetupValidator extends Validator {
     private boolean validate(String configId, SearchCluster searchCluster, String sdName, DeployLogger deployLogger, File tempDir) {
         Instant start = Instant.now();
         try {
+            log.log(Level.FINE, () -> String.format("Validating schema '%s' for %s with config id %s", sdName, searchCluster, configId));
             boolean ret = execValidate(configId, searchCluster, sdName, deployLogger);
             if (!ret) {
                 // Give up, don't say same error msg repeatedly
                 deleteTempDir(tempDir);
             }
-            log.log(Level.FINE, () -> String.format("Validating %s for %s, %s took %s ms",
-                                                    sdName,
-                                                    searchCluster,
-                                                    configId,
-                                                    Duration.between(start, Instant.now()).toMillis()));
+            log.log(Level.FINE, () -> String.format("Validation took %s ms", Duration.between(start, Instant.now()).toMillis()));
             return ret;
         } catch (IllegalArgumentException e) {
             deleteTempDir(tempDir);
@@ -119,45 +116,36 @@ public class RankSetupValidator extends Validator {
     private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException {
         RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder();
         ((RankProfilesConfig.Producer) producer).getConfig(rpcb);
-        RankProfilesConfig rpc = new RankProfilesConfig(rpcb);
-        writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc);
+        writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpcb.build());
 
         IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder();
         ((IndexschemaConfig.Producer) producer).getConfig(iscb);
-        IndexschemaConfig isc = new IndexschemaConfig(iscb);
-        writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc);
+        writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", iscb.build());
 
         AttributesConfig.Builder acb = new AttributesConfig.Builder();
         ((AttributesConfig.Producer) producer).getConfig(acb);
-        AttributesConfig ac = new AttributesConfig(acb);
-        writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac);
+        writeConfig(dir, AttributesConfig.getDefName() + ".cfg", acb.build());
 
         RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder();
         ((RankingConstantsConfig.Producer) producer).getConfig(rccb);
-        RankingConstantsConfig rcc = new RankingConstantsConfig(rccb);
-        writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc);
+        writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rccb.build());
 
         RankingExpressionsConfig.Builder recb = new RankingExpressionsConfig.Builder();
         ((RankingExpressionsConfig.Producer) producer).getConfig(recb);
-        RankingExpressionsConfig rec = new RankingExpressionsConfig(recb);
-        writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", rec);
+        writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", recb.build());
 
         OnnxModelsConfig.Builder omcb = new OnnxModelsConfig.Builder();
         ((OnnxModelsConfig.Producer) producer).getConfig(omcb);
-        OnnxModelsConfig omc = new OnnxModelsConfig(omcb);
-        writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omc);
+        writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omcb.build());
 
         ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder();
         ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb);
-        ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb);
-        writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc);
+        writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifcb.build());
     }
 
     private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException {
-        String configName = "verify-ranksetup.cfg";
-        String configContent = "";
-
         List<String> config = new ArrayList<>();
+
         // Assist verify-ranksetup in finding the actual ONNX model files
         for (OnnxModel model : db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()) {
             String modelPath = getFileRepositoryPath(model.getFilePath().getName(), model.getFileReference());
@@ -168,20 +156,15 @@ public class RankSetupValidator extends Validator {
 
         for (RankExpressionBody expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) {
             int index = config.size() / 2;
+            String modelPath = (expr.getPathType() == DistributableResource.PathType.BLOB)
+                    ? getFileRepositoryPath(expr.getName(), expr.getFileReference())
+                    : getFileRepositoryPath(expr.getFilePath().getName(), expr.getFileReference());
             config.add(String.format("file[%d].ref \"%s\"", index, expr.getFileReference()));
-            if (expr.getPathType() != DistributableResource.PathType.BLOB) {
-                String modelPath = getFileRepositoryPath(expr.getFilePath().getName(), expr.getFileReference());
-                config.add(String.format("file[%d].path \"%s\"", index, modelPath));
-            } else {
-                String modelPath = getFileRepositoryPath(expr.getName(), expr.getFileReference());
-                config.add(String.format("file[%d].path \"%s\"", index, modelPath));
-            }
+            config.add(String.format("file[%d].path \"%s\"", index, modelPath));
         }
 
-        if ( ! config.isEmpty() ) {
-            configContent = StringUtilities.implodeMultiline(config);
-        }
-        IOUtils.writeFile(dir + configName, configContent, false);
+        String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config);
+        IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false);
     }
 
     public static String getFileRepositoryPath(String name, String fileReference) {
@@ -210,14 +193,15 @@ public class RankSetupValidator extends Validator {
     }
 
     private void validateWarn(Exception e, DeployLogger deployLogger) {
-        String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " +
-                Exceptions.toMessageString(e);
+        String msg = "Unable to execute '" + binaryName +
+                     "', validation of rank expressions will only take place when you start Vespa: " +
+                     Exceptions.toMessageString(e);
         deployLogger.logApplicationPackage(Level.WARNING, msg);
     }
 
     private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) {
-        StringBuilder errMsg = new StringBuilder("For search cluster '").append(sc.getClusterName()).append("', ")
-                .append("search definition '").append(sdName).append("': error in rank setup. Details:\n");
+        StringBuilder errMsg = new StringBuilder("Error in rank setup in schema '").append(sdName)
+                .append("' for content cluster '").append(sc.getClusterName()).append("'.").append(" Details:\n");
         for (String line : output.split("\n")) {
             // Remove debug lines from start script
             if (line.startsWith("debug\t")) continue;