diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-04-22 17:04:57 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-04-22 17:04:57 +0200 |
commit | ca9dde5c3a258faeb46e2cecac9d84f9daa5074f (patch) | |
tree | 1b71f760c7bdc13ce3a5010485b8ef4aa5d93997 /config-model | |
parent | 32220e778c1aa5980f9f87c2a64e23481319a8c9 (diff) | |
parent | d60f83752a4e4507ec1dbe08474ff932ef60c110 (diff) |
Merge pull request #5639 from vespa-engine/balder/reduce-memory-footprint-for-rankprofiles
Balder/reduce memory footprint for rankprofiles
Diffstat (limited to 'config-model')
-rw-r--r-- | config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java | 27 | ||||
-rw-r--r-- | config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java | 37 |
2 files changed, 26 insertions, 38 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index b02362154d9..07b79c4f604 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -145,9 +145,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * The rank type definitions used to derive settings for the native rank features */ - private NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); + private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); - private RankProfile rankProfile; + private final Map<String, String> attributeTypes; + private final Map<String, String> queryFeatureTypes; private Set<String> filterFields = new java.util.LinkedHashSet<>(); @@ -155,11 +156,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { * Creates a raw rank profile from the given rank profile */ public Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, AttributeFields attributeFields) { - this.rankProfile = rankProfile.compile(queryProfiles); - deriveRankingFeatures(this.rankProfile); - deriveRankTypeSetting(this.rankProfile, attributeFields); - deriveFilterFields(this.rankProfile); - deriveWeightProperties(this.rankProfile); + RankProfile compiled = rankProfile.compile(queryProfiles); + attributeTypes = compiled.getAttributeTypes(); + queryFeatureTypes = compiled.getQueryFeatureTypes(); + deriveRankingFeatures(compiled); + deriveRankTypeSetting(compiled, attributeFields); + deriveFilterFields(compiled); + deriveWeightProperties(compiled); } private void deriveFilterFields(RankProfile rp) { @@ -209,7 +212,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) { SerializationContext context = new SerializationContext(eMacros); for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) { - String script = e.getValue().getBody().getRoot().toString(context, null, null); + String script = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), script); } return context.serializedFunctions(); @@ -223,7 +226,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { // Is the feature a macro? if (context.getFunction(referenceNode.getName()) != null) { context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), - referenceNode.toString(context, null, null)); + referenceNode.toString(new StringBuilder(), context, null, null).toString()); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block @@ -363,7 +366,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + ""); properties.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + ""); properties.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + ""); - RankProfile.DiversitySettings diversitySettings = rankProfile.getMatchPhaseSettings().getDiversity(); + RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); if (diversitySettings != null) { properties.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute()); properties.put("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups())); @@ -388,10 +391,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer { String fieldName = (String) filterFieldsIterator.next(); properties.put("vespa.isfilterfield." + fieldName + ".part42", String.valueOf(true)); } - for (Map.Entry<String, String> attributeType : rankProfile.getAttributeTypes().entrySet()) { + for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { properties.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue()); } - for (Map.Entry<String, String> queryFeatureType : rankProfile.getQueryFeatureTypes().entrySet()) { + for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) { properties.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue()); } if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index be76a8dfa40..83c6374fe9a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -133,38 +133,23 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * @param deployState the global deploy state to use for this model. */ public VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState) throws IOException, SAXException { - this(configModelRegistry, deployState, true, null); - } - - private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) throws IOException, SAXException { super("vespamodel"); this.deployState = deployState; this.validationOverrides = deployState.validationOverrides(); configModelRegistry = new VespaConfigModelRegistry(configModelRegistry); VespaModelBuilder builder = new VespaDomBuilder(); root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this); - if (complete) { // create a a completed, frozen model - configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry); - addServiceClusters(deployState.getApplicationPackage(), builder); - this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); // must happen after the two lines above - setupRouting(); - this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); - getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState); - freezeModelTopology(); - root.prepare(configModelRepo); - configModelRepo.prepareConfigModels(); - validateWrapExceptions(); - this.deployState = null; - } - else { // create a model with no services instantiated and the given file distributor - this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); - this.fileDistributor = fileDistributor; - } - } - - /** Creates a mutable model with no services instantiated */ - public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry(), null)); + configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry); + addServiceClusters(deployState.getApplicationPackage(), builder); + this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); // must happen after the two lines above + setupRouting(); + this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); + getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState); + freezeModelTopology(); + root.prepare(configModelRepo); + configModelRepo.prepareConfigModels(); + validateWrapExceptions(); + this.deployState = null; } private void validateWrapExceptions() { |