diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-08-21 17:59:08 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-08-21 17:59:08 +0200 |
commit | 6464389e9f980ee1a8d71a075262039939ae1094 (patch) | |
tree | 827b34b4507d6d90a6684dbe92acdac9bc94ab72 | |
parent | 3b426b943a6762f21945d5157e746a2ec011421b (diff) | |
parent | fda447aef5ef5bbd6e3ed35e6d2719f9c59f02fc (diff) |
Merge pull request #6642 from vespa-engine/bratseth/generate-rank-profiles-for-all-models-part-2-6
Import all signaturees
8 files changed, 23 insertions, 43 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java index a59de09af65..99e3ae8abdc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java @@ -79,13 +79,12 @@ public class ConvertedModel { */ public ConvertedModel(Path modelPath, RankProfileTransformContext context, - ImportedModels importedModels, - FeatureArguments arguments) { // TODO: Remove + ImportedModels importedModels) { this.modelPath = modelPath; this.modelName = toModelName(modelPath); ModelStore store = new ModelStore(context.rankProfile().getSearch().sourceApplication(), modelPath); if ( store.hasSourceModel()) - expressions = convertModel(store, context.rankProfile(), context.queryProfiles(), importedModels, arguments); + expressions = convertModel(store, context.rankProfile(), context.queryProfiles(), importedModels); else expressions = transformFromStoredModel(store, context.rankProfile()); } @@ -93,10 +92,9 @@ public class ConvertedModel { private Map<String, RankingExpression> convertModel(ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, - ImportedModels importedModels, - FeatureArguments arguments) { + ImportedModels importedModels) { ImportedModel model = importedModels.imported(store.modelFiles.modelName(), store.sourceModelDir()); - return transformFromImportedModel(model, store, profile, queryProfiles, arguments); + return transformFromImportedModel(model, store, profile, queryProfiles); } /** Returns the expression matching the given arguments */ @@ -136,8 +134,7 @@ public class ConvertedModel { private Map<String, RankingExpression> transformFromImportedModel(ImportedModel model, ModelStore store, RankProfile profile, - QueryProfileRegistry queryProfiles, - FeatureArguments arguments) { + QueryProfileRegistry queryProfiles) { // Add constants Set<String> constantsReplacedByMacros = new HashSet<>(); model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); @@ -150,10 +147,7 @@ public class ConvertedModel { // Add expressions Map<String, RankingExpression> expressions = new HashMap<>(); for (Map.Entry<String, ImportedModel.Signature> signatureEntry : model.signatures().entrySet()) { - if ( ! matches(signatureEntry.getValue(), arguments, Optional.empty())) continue; - for (Map.Entry<String, String> outputEntry : signatureEntry.getValue().outputs().entrySet()) { - if ( ! matches(signatureEntry.getValue(), arguments, Optional.of(outputEntry.getKey()))) continue; addExpression(model.expressions().get(outputEntry.getValue()), modelName + "." + signatureEntry.getKey() + "." + outputEntry.getKey(), constantsReplacedByMacros, @@ -196,13 +190,6 @@ public class ConvertedModel { return expressions; } - private boolean matches(ImportedModel.Signature signature, FeatureArguments arguments, Optional<String> output) { - if ( ! modelName.equals(arguments.modelName)) return false; - if ( arguments.signature.isPresent() && ! signature.name().equals(arguments.signature().get())) return false; - if (output.isPresent() && arguments.output().isPresent() && ! output.get().matches(arguments.output().get())) return false; - return true; - } - private void addExpression(RankingExpression expression, String expressionName, Set<String> constantsReplacedByMacros, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java index 8005608f070..51c44f4066e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java @@ -45,7 +45,7 @@ public class OnnxFeatureConverter extends ExpressionTransformer<RankProfileTrans try { Path modelPath = Path.fromString(ConvertedModel.FeatureArguments.asString(feature.getArguments().expressions().get(0))); // TODO: Increase scope of this instance to a rank profile: - ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels, new ConvertedModel.FeatureArguments(feature.getArguments())); + ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels); return convertedModel.expression(asFeatureArguments(feature.getArguments())); } catch (IllegalArgumentException | UncheckedIOException e) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index 8e22146b381..1343af1b6ec 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -38,7 +38,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil try { Path modelPath = Path.fromString(ConvertedModel.FeatureArguments.asString(feature.getArguments().expressions().get(0))); // TODO: Increase scope of this instance to a rank profile: - ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels, new ConvertedModel.FeatureArguments(feature.getArguments())); + ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels); return convertedModel.expression(asFeatureArguments(feature.getArguments())); } catch (IllegalArgumentException | UncheckedIOException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index a7465fa9695..80629172fe3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -165,8 +165,7 @@ public class RankingExpressionWithOnnxTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx','y'): " + - "No expressions available in model 'mnist_softmax.onnx'", -// "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: mnist_softmax.onnx.default.add", + "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: mnist_softmax.onnx.default.add", Exceptions.toMessageString(expected)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 9a96555bb78..d9bf23ec8fb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -200,9 +200,8 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved','serving_defaultz'): " + - "No expressions available in model 'mnist_softmax_saved'", -// "No expressions named 'serving_defaultz' in model 'mnist_softmax/saved'. "+ -// "Available expressions: mnist_softmax_saved.serving_default.y", + "No expressions named 'serving_defaultz' in model 'mnist_softmax/saved'. "+ + "Available expressions: mnist_softmax_saved.serving_default.y", Exceptions.toMessageString(expected)); } } @@ -218,9 +217,8 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved','serving_default','x'): " + - "No expressions available in model 'mnist_softmax_saved'", -// "No expression 'mnist_softmax_saved.serving_default.x' in model 'mnist_softmax/saved'. " + -// "Available expressions: mnist_softmax_saved.serving_default.y", + "No expression 'mnist_softmax_saved.serving_default.x' in model 'mnist_softmax/saved'. " + + "Available expressions: mnist_softmax_saved.serving_default.y", Exceptions.toMessageString(expected)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java index 7c962e5b6be..0397678fd41 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -46,4 +46,5 @@ public class HttpGetConfigHandler extends HttpHandler { if (config == null) HttpConfigRequest.throwModelNotReady(); return config; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java index 1978c2b241a..3c6a20ff58f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java @@ -24,9 +24,10 @@ import edu.umd.cs.findbugs.annotations.NonNull; /** * A transposed view for cloud.config.model. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class ServiceModel { + private static final String CLUSTERCONTROLLER_TYPENAME = "container-clustercontroller"; private static final String CONTENT_CLUSTER_TYPENAME = "content"; @@ -61,9 +62,8 @@ public final class ServiceModel { List<String> identifiers = s.getIdentifiers(); for (String identifier : identifiers) { if (seenIdentifiers.containsKey(identifier)) { - throw new RuntimeException( - "Congrats, you have a publishable result. We have a very unexpected hash collision" + " between " - + seenIdentifiers.get(identifier) + " and " + s + "."); + throw new RuntimeException("Hash collision" + " between " + + seenIdentifiers.get(identifier) + " and " + s + "."); } seenIdentifiers.put(identifier, s); } @@ -74,9 +74,8 @@ public final class ServiceModel { servicesMap = servicesBuilder.build(); } - private static void addService(Table<String, String, List<Service>> services, - String hostName, - com.yahoo.vespa.serviceview.bindings.Service s) { + private static void addService(Table<String, String, List<Service>> services, String hostName, + com.yahoo.vespa.serviceview.bindings.Service s) { boolean hasStateApi = false; int statePort = 0; List<Integer> ports = new ArrayList<>(s.ports.size()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 26df4efa3c1..3391eee4d62 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -172,18 +172,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) - return JobControllerApiHandlerHelper.jobTypeResponse(jobTypes(path), latestRuns(path), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) - return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) - return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(jobTypes(path), latestRuns(path), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) - return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); return ErrorResponse.notFoundError("Nothing at " + path); } |