diff options
105 files changed, 521 insertions, 277 deletions
diff --git a/component/abi-spec.json b/component/abi-spec.json index 0e33fde3c5a..4e89b0e717c 100644 --- a/component/abi-spec.json +++ b/component/abi-spec.json @@ -55,9 +55,9 @@ "final" ], "methods": [ + "public void <init>(java.lang.String)", "public void <init>(java.lang.String, com.yahoo.component.Version, com.yahoo.component.ComponentId)", "public void <init>(java.lang.String, com.yahoo.component.Version)", - "public void <init>(java.lang.String)", "public com.yahoo.component.ComponentId nestInNamespace(com.yahoo.component.ComponentId)", "public java.lang.String getName()", "public com.yahoo.component.Version getVersion()", @@ -75,6 +75,7 @@ "public java.lang.String toFileName()", "public static com.yahoo.component.ComponentId fromFileName(java.lang.String)", "public static void resetGlobalCountersForTests()", + "public static com.yahoo.component.ComponentId newAnonymous(java.lang.String)", "public bridge synthetic int compareTo(java.lang.Object)" ], "fields": [] diff --git a/component/src/main/java/com/yahoo/component/AbstractComponent.java b/component/src/main/java/com/yahoo/component/AbstractComponent.java index 254ef0e5db9..3fbae31a743 100644 --- a/component/src/main/java/com/yahoo/component/AbstractComponent.java +++ b/component/src/main/java/com/yahoo/component/AbstractComponent.java @@ -60,7 +60,7 @@ public class AbstractComponent implements Component, Deconstructable { return id; } - //This should only happen in tests, so thread safety should not be an issue. + // This should only happen in tests, so thread safety should not be an issue. private void setTestId() { id = ComponentId.createAnonymousComponentId("test_" + getClass().getName()); } diff --git a/component/src/main/java/com/yahoo/component/ComponentId.java b/component/src/main/java/com/yahoo/component/ComponentId.java index a19c70895ba..7320d24b57c 100644 --- a/component/src/main/java/com/yahoo/component/ComponentId.java +++ b/component/src/main/java/com/yahoo/component/ComponentId.java @@ -13,31 +13,17 @@ import java.util.concurrent.atomic.AtomicInteger; */ public final class ComponentId implements Comparable<ComponentId> { - private final class VersionHandler implements Spec.VersionHandler<Version> { - - @Override - public Version emptyVersion() { - return Version.emptyVersion; - } - - @Override - public int compare(Version v1, Version v2) { - return v1.compareTo(v2); - } - } - private final Spec<Version> spec; private final boolean anonymous; - private final static class Counter { - private int count = 0; - public int getAndIncrement() { return count++; } - } + + private static AtomicInteger threadIdCounter = new AtomicInteger(0); + private static ThreadLocal<Counter> threadLocalUniqueId = new ThreadLocal<Counter>() { @Override protected Counter initialValue() { return new Counter(); } }; - private static AtomicInteger threadIdCounter = new AtomicInteger(0); + private static ThreadLocal<String> threadId = new ThreadLocal<String>() { @Override protected String initialValue() { return new String("_" + threadIdCounter.getAndIncrement() + "_"); @@ -47,18 +33,17 @@ public final class ComponentId implements Comparable<ComponentId> { /** Precomputed string value */ private final String stringValue; - private ComponentId(String name, Version version, ComponentId namespace, boolean anonymous) { - if (anonymous) { - name = createAnonymousName(name); - } - spec = new Spec<>(new VersionHandler(), name, version, namespace); - this.anonymous = anonymous; - - stringValue = spec.createStringValue(); + /** + * Creates a component id from the id string form: name(:version)?(@namespace)?, + * where version has the form 1(.2(.3(.identifier)?)?)? + * and namespace is a component id + */ + public ComponentId(String id) { + this(new SpecSplitter(id)); } - private String createAnonymousName(String name) { - return new StringBuilder(name).append(threadId.get()).append(threadLocalUniqueId.get().getAndIncrement()).toString(); + private ComponentId(SpecSplitter splitter) { + this(splitter.name, Version.fromString(splitter.version), splitter.namespace); } public ComponentId(String name, Version version, ComponentId namespace) { @@ -70,17 +55,10 @@ public final class ComponentId implements Comparable<ComponentId> { this(name, version, null); } - /** - * Creates a component id from the id string form: name(:version)?(@namespace)?, - * where version has the form 1(.2(.3(.identifier)?)?)? - * and namespace is a component id - */ - public ComponentId(String id) { - this(new SpecSplitter(id)); - } - - private ComponentId(SpecSplitter splitter) { - this(splitter.name, Version.fromString(splitter.version), splitter.namespace); + private ComponentId(String id, Version version, ComponentId namespace, boolean anonymous) { + this.spec = new Spec<>(new VersionHandler(), id, version, namespace); + this.anonymous = anonymous; + this.stringValue = spec.createStringValue(); } public ComponentId nestInNamespace(ComponentId namespace) { @@ -116,12 +94,13 @@ public final class ComponentId implements Comparable<ComponentId> { return spec.toString(); } + @Override public boolean equals(Object o) { - if (o==this) return true; + if (o == this) return true; if ( ! (o instanceof ComponentId)) return false; ComponentId c = (ComponentId) o; - if (isAnonymous() || c.isAnonymous()) + if (isAnonymous() || c.isAnonymous()) // TODO: Stop doing this return false; return c.stringValue().equals(stringValue); @@ -150,7 +129,7 @@ public final class ComponentId implements Comparable<ComponentId> { /** Creates a componentId that is unique for this run-time instance */ public static ComponentId createAnonymousComponentId(String baseName) { - return new ComponentId(baseName, null, null, true); + return new ComponentId(createAnonymousId(baseName), null, null, true); } public boolean isAnonymous() { @@ -232,4 +211,35 @@ public final class ComponentId implements Comparable<ComponentId> { threadLocalUniqueId.set(new Counter()); } + private static String createAnonymousId(String name) { + return name + threadId.get() + threadLocalUniqueId.get().getAndIncrement(); + } + + /** Creates a component id with the given value, marked as anonymous */ + public static ComponentId newAnonymous(String spec) { + var splitter = new SpecSplitter(spec); + return new ComponentId(splitter.name, Version.fromString(splitter.version), splitter.namespace, true); + } + + private final class VersionHandler implements Spec.VersionHandler<Version> { + + @Override + public Version emptyVersion() { + return Version.emptyVersion; + } + + @Override + public int compare(Version v1, Version v2) { + return v1.compareTo(v2); + } + + } + + private final static class Counter { + + private int count = 0; + public int getAndIncrement() { return count++; } + + } + } diff --git a/component/src/main/java/com/yahoo/component/provider/ComponentRegistry.java b/component/src/main/java/com/yahoo/component/provider/ComponentRegistry.java index 9aa677737d6..ec6f21e4f53 100644 --- a/component/src/main/java/com/yahoo/component/provider/ComponentRegistry.java +++ b/component/src/main/java/com/yahoo/component/provider/ComponentRegistry.java @@ -28,10 +28,10 @@ import java.util.Set; public class ComponentRegistry<COMPONENT> { /** All versions of all components, indexed by name and namespace */ - private Map<ComponentId, Map<String, Map<Version, COMPONENT>>> componentsByNameByNamespace = new LinkedHashMap<>(); + private final Map<ComponentId, Map<String, Map<Version, COMPONENT>>> componentsByNameByNamespace = new LinkedHashMap<>(); /** All versions of all components indexed by id */ - private Map<ComponentId, COMPONENT> componentsById =new LinkedHashMap<>(); + private final Map<ComponentId, COMPONENT> componentsById =new LinkedHashMap<>(); /** True when this cannot be changed any more */ private boolean frozen = false; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index d51427abd90..9ad257fad04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -36,6 +36,7 @@ public final class ApplicationContainer extends Container implements addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.SessionCache")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider")); + addComponent(new SimpleComponent("com.yahoo.container.jdisc.ZoneInfoProvider")); } @Override diff --git a/config-model/src/test/derived/neuralnet/query-profiles.cfg b/config-model/src/test/derived/neuralnet/query-profiles.cfg index 817640a89fd..94246ccd96b 100644 --- a/config-model/src/test/derived/neuralnet/query-profiles.cfg +++ b/config-model/src/test/derived/neuralnet/query-profiles.cfg @@ -5,20 +5,20 @@ queryprofiletype[].strict false queryprofiletype[].matchaspath false queryprofiletype[].inherit[] "native" queryprofiletype[].field[].name "ranking" -queryprofiletype[].field[].type "query-profile:ranking_0_0" +queryprofiletype[].field[].type "query-profile:ranking_0" queryprofiletype[].field[].overridable true queryprofiletype[].field[].mandatory false queryprofiletype[].field[].alias "" -queryprofiletype[].id "ranking_0_0" +queryprofiletype[].id "ranking_0" queryprofiletype[].strict false queryprofiletype[].matchaspath false queryprofiletype[].inherit[] "ranking" queryprofiletype[].field[].name "features" -queryprofiletype[].field[].type "query-profile:features_0_1" +queryprofiletype[].field[].type "query-profile:features_1" queryprofiletype[].field[].overridable true queryprofiletype[].field[].mandatory false queryprofiletype[].field[].alias "rankfeature" -queryprofiletype[].id "features_0_1" +queryprofiletype[].id "features_1" queryprofiletype[].strict false queryprofiletype[].matchaspath false queryprofiletype[].field[].name "query(W_0)" diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java index ffd342f36cd..44daec42b88 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java @@ -198,7 +198,7 @@ class ServletOutputStreamWriter { final int bytesToSend = buffer.remaining(); try { if (buffer.hasArray()) { - outputStream.write(buffer.array(), buffer.arrayOffset(), buffer.remaining()); + outputStream.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); } else { final byte[] array = new byte[buffer.remaining()]; buffer.get(array); diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java index 5b341e0a3e6..d4b0771f482 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java @@ -81,5 +81,4 @@ class Utils { client.start(); return client; } - } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ZoneInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ZoneInfoProvider.java new file mode 100644 index 00000000000..30a4c740ff0 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ZoneInfoProvider.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import ai.vespa.cloud.Environment; +import ai.vespa.cloud.Zone; +import ai.vespa.cloud.ZoneInfo; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.di.componentgraph.Provider; + +/** + * Provides information about the zone in which this container is running. + * This is available and can be injected when running in a cloud environment. + * + * @author bratseth + */ +public class ZoneInfoProvider extends AbstractComponent implements Provider<ZoneInfo> { + + private final ZoneInfo instance; + + @Inject + public ZoneInfoProvider(ConfigserverConfig csConfig) { + this.instance = new ZoneInfo(new Zone(Environment.valueOf(csConfig.environment()), csConfig.region())); + } + + @Override + public ZoneInfo get() { return instance; } + +} diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 8ebb11cc8c5..6694a37469b 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5840,8 +5840,8 @@ "public" ], "methods": [ - "protected void <init>()", - "protected void <init>(java.lang.String)", + "protected void <init>(com.yahoo.search.query.profile.QueryProfileRegistry)", + "protected void <init>(java.lang.String, com.yahoo.search.query.profile.QueryProfileRegistry)", "protected java.lang.Object checkAndConvertAssignment(java.lang.String, java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", "protected com.yahoo.search.query.profile.QueryProfile createSubProfile(java.lang.String, com.yahoo.search.query.profile.DimensionBinding)", "public com.yahoo.search.query.profile.OverridableQueryProfile clone()", @@ -5865,7 +5865,9 @@ "public void <init>(com.yahoo.component.ComponentId)", "public void <init>(java.lang.String)", "public void <init>(com.yahoo.component.ComponentId, java.lang.String)", + "public void <init>(com.yahoo.component.ComponentId, java.lang.String, com.yahoo.search.query.profile.QueryProfileRegistry)", "public java.lang.String getSource()", + "protected final com.yahoo.search.query.profile.QueryProfileRegistry getOwner()", "public com.yahoo.search.query.profile.types.QueryProfileType getType()", "public void setType(com.yahoo.search.query.profile.types.QueryProfileType)", "public com.yahoo.search.query.profile.QueryProfileVariants getVariants()", @@ -5923,7 +5925,9 @@ "public bridge synthetic com.yahoo.component.AbstractComponent clone()", "public bridge synthetic java.lang.Object clone()" ], - "fields": [] + "fields": [ + "public final com.yahoo.search.query.profile.QueryProfileRegistry owner" + ] }, "com.yahoo.search.query.profile.QueryProfileCompiler": { "superClass": "java.lang.Object", @@ -5973,7 +5977,8 @@ "public com.yahoo.search.query.profile.types.QueryProfileTypeRegistry getTypeRegistry()", "public com.yahoo.search.query.profile.QueryProfile findQueryProfile(java.lang.String)", "public void freeze()", - "public com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry compile()" + "public com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry compile()", + "public com.yahoo.component.ComponentId createAnonymousId(java.lang.String)" ], "fields": [] }, @@ -6498,7 +6503,8 @@ "public void register(com.yahoo.search.query.profile.types.QueryProfileType)", "public boolean hasApplicationTypes()", "public void freeze()", - "public static com.yahoo.search.query.profile.types.QueryProfileTypeRegistry emptyFrozen()" + "public static com.yahoo.search.query.profile.types.QueryProfileTypeRegistry emptyFrozen()", + "public com.yahoo.component.ComponentId createAnonymousId(java.lang.String)" ], "fields": [] }, diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java index 08bdca78ee6..8f22c6902c3 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java @@ -2,20 +2,17 @@ package com.yahoo.search.query.profile; import com.yahoo.processing.request.CompoundName; -import com.yahoo.protect.Validator; import java.util.HashMap; import java.util.List; import java.util.Map; /** - * <p>A wrapper of a query profile where overrides to the values in the referenced - * profile can be set.</p> - * - * <p>This is used to allow configured overrides (in a particular referencing profile) of a referenced query profile. - * - * <p>Properties which are defined as not overridable in the type (if any) of the referenced query profile - * cannot be set.</p> + * A wrapper of a query profile where overrides to the values in the referenced + * profile can be set. + * This is used to allow configured overrides (in a particular referencing profile) of a referenced query profile. + * Properties which are defined as not overridable in the type (if any) of the referenced query profile + * cannot be set. * * @author bratseth */ @@ -31,7 +28,7 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple * @param backingProfile the backing profile, which is assumed read only, never null */ public BackedOverridableQueryProfile(QueryProfile backingProfile) { - Validator.ensureNotNull("An overridable query profile must be backed by a real query profile", backingProfile); + super(backingProfile.getOwner()); setType(backingProfile.getType()); this.backingProfile = backingProfile; } @@ -66,7 +63,7 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple if (backing instanceof QueryProfile) return new BackedOverridableQueryProfile((QueryProfile)backing); else - return new OverridableQueryProfile(); // Nothing is set in this branch, so nothing to override, but need override checking + return new OverridableQueryProfile(getOwner()); // Nothing is set in this branch, so nothing to override, but need override checking } /** Returns a clone of this which can be independently overridden, but which refers to the same backing profile */ diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java index b8de533dfbe..ac0712eeb63 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java @@ -16,12 +16,12 @@ public class OverridableQueryProfile extends QueryProfile { private static final String simpleClassName = OverridableQueryProfile.class.getSimpleName(); /** Creates an unbacked overridable query profile */ - protected OverridableQueryProfile() { - this(""); + protected OverridableQueryProfile(QueryProfileRegistry owner) { + this("", owner); } - protected OverridableQueryProfile(String sourceName) { - super(ComponentId.createAnonymousComponentId(simpleClassName), sourceName); + protected OverridableQueryProfile(String sourceName, QueryProfileRegistry owner) { + super(createAnonymousId(owner), sourceName, owner); } @Override @@ -35,7 +35,8 @@ public class OverridableQueryProfile extends QueryProfile { @Override protected QueryProfile createSubProfile(String name, DimensionBinding binding) { - return new OverridableQueryProfile(getSource()); // Nothing is set in this branch, so nothing to override, but need override checking + // Nothing is set in this branch, so nothing to override, but need override checking + return new OverridableQueryProfile(getSource(), getOwner()); } /** Returns a clone of this which can be independently overridden */ @@ -43,7 +44,7 @@ public class OverridableQueryProfile extends QueryProfile { public OverridableQueryProfile clone() { if (isFrozen()) return this; OverridableQueryProfile clone = (OverridableQueryProfile)super.clone(); - clone.initId(ComponentId.createAnonymousComponentId(simpleClassName)); + clone.initId(createAnonymousId(getOwner())); return clone; } @@ -52,4 +53,9 @@ public class OverridableQueryProfile extends QueryProfile { return "an overridable query profile with no backing"; } + private static ComponentId createAnonymousId(QueryProfileRegistry owner) { + return owner != null ? owner.createAnonymousId(simpleClassName) + : ComponentId.createAnonymousComponentId(simpleClassName); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index ee7d44f457f..c30a78da57d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -45,6 +45,9 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** The name of the source of this (a file) */ private final String source; + /** The query profile registry owning this, or null if none (which will only happen in tests) */ + public final QueryProfileRegistry owner; + /** Defines the permissible content of this, or null if any content is permissible */ private QueryProfileType type = null; @@ -84,8 +87,13 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } public QueryProfile(ComponentId id, String sourceName) { + this(id, sourceName, null); + } + + public QueryProfile(ComponentId id, String sourceName, QueryProfileRegistry owner) { super(id); this.source = sourceName; + this.owner = owner; if ( ! id.isAnonymous()) validateName(id.getName()); } @@ -96,6 +104,8 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable public String getSource() { return source; } + protected final QueryProfileRegistry getOwner() { return owner; } + /** Returns the type of this or null if it has no type */ public QueryProfileType getType() { return type; } @@ -144,6 +154,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** * Returns the content fields declared in this (i.e not including those inherited) as a read-only map. + * * @throws IllegalStateException if this is frozen */ public Map<String, Object> declaredContent() { @@ -390,7 +401,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * Switches this from write-only to read-only mode. * This profile can never be modified again after this method returns. * Calling this on an already frozen profile has no effect. - * <p> + * * Calling this will also freeze any profiles inherited and referenced by this. */ // TODO: Remove/simplify as query profiles are not used at query time @@ -624,7 +635,8 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable QueryProfile newProfile = (QueryProfile)newValue; if ( ! (existingValue instanceof QueryProfile)) { if ( ! isModifiable(newProfile)) { - newProfile = new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable + // Make the query profile reference overridable + newProfile = new BackedOverridableQueryProfile(newProfile); } newProfile.value = existingValue; return newProfile; @@ -652,7 +664,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } } - private static QueryProfile combineProfiles(QueryProfile newProfile,QueryProfile existingProfile) { + private static QueryProfile combineProfiles(QueryProfile newProfile, QueryProfile existingProfile) { QueryProfile returnValue = null; QueryProfile existingModifiable; @@ -720,7 +732,8 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * This default implementation returns an empty profile. */ protected QueryProfile createSubProfile(String name, DimensionBinding dimensionBinding) { - return new QueryProfile(ComponentId.createAnonymousComponentId(name), source); + var id = owner != null ? owner.createAnonymousId(name) : ComponentId.createAnonymousComponentId(name); + return new QueryProfile(id, source, owner); } /** Do a variant-aware content lookup in this */ diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java index 83b3cd8c4d0..1910078058a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile; +import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import java.util.concurrent.atomic.AtomicInteger; + /** * A set of query profiles. This also holds the query profile types as a dependent registry * @@ -14,7 +17,9 @@ import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; */ public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> { - private QueryProfileTypeRegistry queryProfileTypeRegistry = new QueryProfileTypeRegistry(); + private int nextAnonymousId = 0; + + private final QueryProfileTypeRegistry queryProfileTypeRegistry = new QueryProfileTypeRegistry(); /** Register this type by its id */ public void register(QueryProfile profile) { @@ -83,4 +88,8 @@ public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> { public CompiledQueryProfileRegistry compile() { return QueryProfileCompiler.compile(this); } + public ComponentId createAnonymousId(String name) { + return ComponentId.newAnonymous(name + "_" + (nextAnonymousId++)); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java index f840885f2e4..ceeed9d9167 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -74,7 +74,7 @@ public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber } private static void createProfile(QueryProfilesConfig.Queryprofile config, QueryProfileRegistry registry) { - QueryProfile profile = new QueryProfile(new ComponentId(config.id()), config.id()); + QueryProfile profile = new QueryProfile(new ComponentId(config.id()), config.id(), registry); try { String typeId = config.type(); if (typeId != null && ! typeId.isEmpty()) diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java index 555d3a8e6b7..c94987ce88e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java @@ -135,7 +135,7 @@ public class QueryProfileXMLReader { ComponentId id = new ComponentId(idString); validateFileNameToId(reader.getName(), id, "query profile"); - QueryProfile queryProfile = new QueryProfile(id, reader.getName()); + QueryProfile queryProfile = new QueryProfile(id, reader.getName(), registry); String typeId = root.getAttribute("type"); if (typeId != null && ! typeId.equals("")) { QueryProfileType type = registry.getType(typeId); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index c9fcb854771..8b6470996d5 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -321,13 +321,13 @@ public class QueryProfileType extends FreezableSimpleComponent { // found in registry but not already added in *this* type (getField also checks parents): extend it if (type != null && ! fields.containsKey(name)) { - type = new QueryProfileType(ComponentId.createAnonymousComponentId(type.getIdString()), + type = new QueryProfileType(registry.createAnonymousId(type.getIdString()), new HashMap<>(), List.of(type)); } if (type == null) { // create it - type = new QueryProfileType(ComponentId.createAnonymousComponentId(name)); + type = new QueryProfileType(registry.createAnonymousId(name)); } if (fieldDescription == null) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java index 58a1ae7ad14..201f246b30c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.search.Query; import com.yahoo.search.query.profile.QueryProfileRegistry; @@ -12,6 +13,8 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; */ public class QueryProfileTypeRegistry extends ComponentRegistry<QueryProfileType> { + private int nextAnonymousId = 0; + private final int nativeProfileCount; public QueryProfileTypeRegistry() { @@ -42,4 +45,8 @@ public class QueryProfileTypeRegistry extends ComponentRegistry<QueryProfileType return registry; } + public ComponentId createAnonymousId(String name) { + return ComponentId.newAnonymous(name + "_" + (nextAnonymousId++)); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java index e04835c01b2..c47e1e5b23c 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -449,6 +449,24 @@ public class XmlReadingTestCase { query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default")); assertEquals("MyProfile2", query.properties().get("profileRef.name")); } + + } + + @Test + public void testAnonymousIdsAreStableBetweenImports() { + QueryProfileRegistry registry1 = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance"); + var childIn1 = registry1.findQueryProfile("child"); + var childTypeIn1 = registry1.getType("childType"); + + QueryProfileRegistry registry2 = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance"); + var childIn2 = registry2.findQueryProfile("child"); + var childTypeIn2 = registry2.getType("childType"); + + assertEquals(((QueryProfile)childIn1.lookup("a", Map.of())).getId().stringValue(), + ((QueryProfile)childIn2.lookup("a", Map.of())).getId().stringValue()); + + assertEquals(childTypeIn1.getType("a").getId().stringValue(), + childTypeIn2.getType("a").getId().stringValue()); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/child.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/child.xml new file mode 100644 index 00000000000..20186c440df --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/child.xml @@ -0,0 +1,5 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="child" inherits="parent" type="childType"> + <field name="a.b.c">a.b.c-child</field> +</query-profile> + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/parent.xml new file mode 100644 index 00000000000..21a19eda24f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/parent.xml @@ -0,0 +1,5 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent" type="parentType"> + <field name="a.b">a.b-parent</field> + <field name="a.b.c">a.b.c-parent</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/types/childType.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/types/childType.xml new file mode 100644 index 00000000000..18665d195b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/types/childType.xml @@ -0,0 +1,4 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="childType" inherits="parentType"> + <field name="a.b.c" type="string"/> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/types/parentType.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/types/parentType.xml new file mode 100644 index 00000000000..771c373df12 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typedinheritance/types/parentType.xml @@ -0,0 +1,3 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="parentType"> +</query-profile-type> diff --git a/dist/vespa.spec b/dist/vespa.spec index e6eff61b214..fcfe13fedd4 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -278,6 +278,7 @@ Requires: gtest %define _skip_vespamalloc 1 %endif Requires: %{name}-base = %{version}-%{release} +Requires: %{name}-base-libs = %{version}-%{release} Requires: %{name}-libs = %{version}-%{release} Requires: %{name}-clients = %{version}-%{release} Requires: %{name}-config-model-fat = %{version}-%{release} @@ -464,6 +465,7 @@ Vespa - The open big data serving engine - tools Summary: Vespa - The open big data serving engine - ann-benchmark +Requires: %{name}-base-libs = %{version}-%{release} Requires: %{name}-libs = %{version}-%{release} %if 0%{?el7} Requires: python3 diff --git a/hosted-zone-api/abi-spec.json b/hosted-zone-api/abi-spec.json index 4195fd5f10c..6a6da57a2a1 100644 --- a/hosted-zone-api/abi-spec.json +++ b/hosted-zone-api/abi-spec.json @@ -78,5 +78,17 @@ "public static ai.vespa.cloud.Zone from(java.lang.String)" ], "fields": [] + }, + "ai.vespa.cloud.ZoneInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.cloud.Zone)", + "public ai.vespa.cloud.Zone zone()" + ], + "fields": [] } }
\ No newline at end of file diff --git a/hosted-zone-api/src/main/java/ai/vespa/cloud/ZoneInfo.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/ZoneInfo.java new file mode 100644 index 00000000000..d9af2421ab9 --- /dev/null +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/ZoneInfo.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.cloud; + +import java.util.Objects; + +/** + * Provides information about the zone in which this container is running. + * This is available and can be injected when running in a cloud environment. + * If you don't need any other information than the zone this should be preferred + * to SystemInfo as it will never change at runtime and therefore does not + * cause unnecessary reconstruction. + * + * @author bratseth + */ +public class ZoneInfo { + + private final Zone zone; + + public ZoneInfo(Zone zone) { + Objects.requireNonNull(zone, "Zone cannot be null!"); + this.zone = zone; + } + + /** Returns the zone this is running in */ + public Zone zone() { return zone; } + +} diff --git a/sd-plugin/build.gradle b/sd-plugin/build.gradle index 5a0c37dbf33..90a137ce473 100644 --- a/sd-plugin/build.gradle +++ b/sd-plugin/build.gradle @@ -1,4 +1,4 @@ -// This "noinspection" comment is here to fix a warning +// This "noinspection" comment below is here to fix a warning //noinspection GroovyAssignabilityCheck plugins { id 'org.jetbrains.intellij' version '1.1.4' @@ -35,12 +35,12 @@ compileJava { } -group 'org.vz.native' -version '1.0.1' +group 'org.yahoo.native' +version '1.0.0' sourceCompatibility = 11 -// This "noinspection" comment is here to fix a warning +// This "noinspection" comment below is here to fix a warning //noinspection GroovyAssignabilityCheck repositories { mavenCentral() diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java index f313bb2be59..99bebe21b87 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java @@ -8,7 +8,6 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiManager; import com.intellij.psi.search.FileTypeIndex; import com.intellij.psi.search.GlobalSearchScope; -import com.intellij.util.indexing.FileBasedIndex; import org.intellij.sdk.language.psi.SdDeclaration; import org.intellij.sdk.language.psi.SdFile; import org.jetbrains.annotations.NotNull; @@ -19,17 +18,13 @@ import java.util.List; /** * This class is used for the extension (in plugin.xml) to enable "Go To Symbol" feature. - * @author shahariel + * @author Shahar Ariel */ public class SdChooseByNameContributor implements ChooseByNameContributor { @Override public String @NotNull [] getNames(Project project, boolean includeNonProjectItems) { - Collection<VirtualFile> virtualFiles = FileBasedIndex.getInstance().getContainingFiles( - FileTypeIndex.NAME, - SdFileType.INSTANCE, - GlobalSearchScope.allScope(project) - ); + Collection<VirtualFile> virtualFiles = FileTypeIndex.getFiles(SdFileType.INSTANCE, GlobalSearchScope.allScope(project)); List<SdDeclaration> declarations = new ArrayList<>(); @@ -47,11 +42,7 @@ public class SdChooseByNameContributor implements ChooseByNameContributor { @Override public NavigationItem @NotNull [] getItemsByName(String name, String pattern, Project project, boolean includeNonProjectItems) { - Collection<VirtualFile> virtualFiles = FileBasedIndex.getInstance().getContainingFiles( - FileTypeIndex.NAME, - SdFileType.INSTANCE, - GlobalSearchScope.allScope(project) - ); + Collection<VirtualFile> virtualFiles = FileTypeIndex.getFiles(SdFileType.INSTANCE, GlobalSearchScope.allScope(project)); List<SdDeclaration> declarations = new ArrayList<>(); diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java index 89460099790..93c3e7ccc50 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java @@ -6,7 +6,7 @@ import com.intellij.psi.codeStyle.CustomCodeStyleSettings; /** * This class represent a code style settings, and creates an option page in settings/preferences. - * @author shahariel + * @author Shahar Ariel */ public class SdCodeStyleSettings extends CustomCodeStyleSettings { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java index 3b8136f92b3..3b075b0f438 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java @@ -13,7 +13,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdCodeStyleSettings. - * @author shahariel + * @author Shahar Ariel */ public class SdCodeStyleSettingsProvider extends CodeStyleSettingsProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java index 5d9fbfaa07d..d0f28848cf0 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java @@ -7,7 +7,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable turning a line into a comment with * "Code -> Comment with line comment". - * @author shahariel + * @author Shahar Ariel */ public class SdCommenter implements Commenter { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java index 7cf517ba2e9..8bcda614ee7 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java @@ -14,7 +14,7 @@ import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml) to enables Auto-Complete. Partially works for now. - * @author shahariel + * @author Shahar Ariel */ public class SdCompletionContributor extends CompletionContributor { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java index 0f6189abbe6..ccc4c6b6348 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java @@ -9,7 +9,7 @@ import javax.swing.Icon; /** * This class is used for the extension (in plugin.xml), to define SD as a file's type. - * @author shahariel + * @author Shahar Ariel */ public class SdFileType extends LanguageFileType { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java index 21c46f15e55..806fcd6b6ba 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java @@ -7,7 +7,7 @@ import javax.swing.Icon; /** * This class is used for defining Icons for the IDE. - * @author shahariel + * @author Shahar Ariel */ public class SdIcons { public static final Icon FILE = IconLoader.getIcon("icons/sd_icon.png", SdIcons.class); diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java index d283514076d..fa253c4d605 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java @@ -5,7 +5,7 @@ import com.intellij.lang.Language; /** * This class defines SD as a language. - * @author shahariel + * @author Shahar Ariel */ public class SdLanguage extends Language { public static final SdLanguage INSTANCE = new SdLanguage(); diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java index efe4e9ea0ae..24fef3af40c 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java @@ -8,7 +8,7 @@ import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml), to make the IDE use our plugin's code for coding style. - * @author shahariel + * @author Shahar Ariel */ public class SdLanguageCodeStyleSettingsProvider extends LanguageCodeStyleSettingsProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLexerAdapter.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLexerAdapter.java index c9137fa9234..58329222a19 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLexerAdapter.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdLexerAdapter.java @@ -5,7 +5,7 @@ import com.intellij.lexer.FlexAdapter; /** * This class adapts the JFlex lexer to the IntelliJ Platform Lexer API. - * @author shahariel + * @author Shahar Ariel */ public class SdLexerAdapter extends FlexAdapter { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java index 4f602c56576..1b5cba028a7 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java @@ -9,7 +9,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable refactoring. - * @author shahariel + * @author Shahar Ariel */ public class SdRefactoringSupportProvider extends RefactoringSupportProvider { @Override diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java index 3cb41bb6132..bce51ccc2ff 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java @@ -20,7 +20,7 @@ import java.util.List; /** * This class represent a reference to a Psi Element. - * @author shahariel + * @author Shahar Ariel */ public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPolyVariantReference { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java index 31acd5ff730..aaceaaa02cd 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java @@ -17,7 +17,7 @@ import java.util.HashSet; /** * This class defines the syntax highlighting of an SD file. - * @author shahariel + * @author Shahar Ariel */ public class SdSyntaxHighlighter extends SyntaxHighlighterBase { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java index b2ace3156e9..618e188f680 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java @@ -9,7 +9,7 @@ import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml) to the class SdSyntaxHighlighter. - * @author shahariel + * @author Shahar Ariel */ public class SdSyntaxHighlighterFactory extends SyntaxHighlighterFactory { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java index 92e7c9e6f85..5d6cd85489f 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java @@ -11,7 +11,6 @@ import com.intellij.psi.PsiReference; import com.intellij.psi.search.FileTypeIndex; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.util.PsiTreeUtil; -import com.intellij.util.indexing.FileBasedIndex; import org.intellij.sdk.language.psi.SdAnnotationFieldDefinition; import org.intellij.sdk.language.psi.SdArgumentDefinition; import org.intellij.sdk.language.psi.SdDeclaration; @@ -40,7 +39,7 @@ import java.util.List; /** * This is the util class for the plugin's code. - * @author shahariel + * @author Shahar Ariel */ public class SdUtil { @@ -75,8 +74,8 @@ public class SdUtil { } } SdIdentifier ancestorIdentifier = (SdIdentifier) ancestorAST.getPsi(); - return ancestorIdentifier.getReference().resolve(); - + PsiReference ref = ancestorIdentifier.getReference(); + return ref != null ? ref.resolve() : null; } public static String createFunctionDescription(SdFunctionDefinition macro) { @@ -128,12 +127,9 @@ public class SdUtil { if (docName == null) { return result; } - - Collection<VirtualFile> virtualFiles = FileBasedIndex.getInstance().getContainingFiles( - FileTypeIndex.NAME, - SdFileType.INSTANCE, - GlobalSearchScope.allScope(project) - ); + + Collection<VirtualFile> virtualFiles = FileTypeIndex.getFiles(SdFileType.INSTANCE, GlobalSearchScope.allScope(project)); + for (VirtualFile vfile : virtualFiles) { SdFile sdFile = (SdFile) PsiManager.getInstance(project).findFile(vfile); if (sdFile != null && !sdFile.getName().equals(docName + ".sd")) { @@ -166,7 +162,7 @@ public class SdUtil { PsiElement curRankProfile = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class); while (curRankProfile != null) { for (SdFunctionDefinition macro : PsiTreeUtil.collectElementsOfType(curRankProfile, SdFunctionDefinition.class)) { - if (macro.getName().equals(name)) { + if (macro.getName() != null && macro.getName().equals(name)) { result.add(macro); return result; } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java index 8047e198d88..ba081e9d415 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java @@ -15,7 +15,7 @@ import org.jetbrains.annotations.Nullable; /** * This class represent a Document Summary that groups elements in the "Find Usages" window. - * @author shahariel + * @author Shahar Ariel */ public class SdDocumentSummaryGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java index ff519436994..0e6f0358ca5 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java @@ -9,7 +9,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdDocumentSummaryGroupingRule. - * @author shahariel + * @author Shahar Ariel */ public class SdDocumentSummaryGroupingRuleProvider implements FileStructureGroupRuleProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java index 6841f608efd..b48c35e5c49 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java @@ -23,7 +23,7 @@ import java.util.List; /** * This class handles creating the "Find Usages" window. - * @author shahariel + * @author Shahar Ariel */ public class SdFindUsagesHandler extends FindUsagesHandler { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java index 2ccd7f6ea41..b96cad508f7 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java @@ -10,7 +10,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdFindUsagesHandler. - * @author shahariel + * @author Shahar Ariel */ public class SdFindUsagesHandlerFactory extends FindUsagesHandlerFactory { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java index ba71c864ce5..56aaae6d02a 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java @@ -18,7 +18,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable "find Usages" window using the plugin code. - * @author shahariel + * @author Shahar Ariel */ public class SdFindUsagesProvider implements FindUsagesProvider { @Nullable diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java index 77de9fd9534..1fa848313e0 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java @@ -15,7 +15,7 @@ import org.jetbrains.annotations.Nullable; /** * This class represent a Rank Profile that groups elements in the "Find Usages" window. - * @author shahariel + * @author Shahar Ariel */ public class SdRankProfileGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java index 7527b93c947..0b973a56644 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java @@ -9,7 +9,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdRankProfileGroupingRule. - * @author shahariel + * @author Shahar Ariel */ public class SdRankProfileGroupingRuleProvider implements FileStructureGroupRuleProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java index 9042be84d75..0423db89cd6 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java @@ -3,7 +3,6 @@ package org.intellij.sdk.language.findUsages; import com.intellij.navigation.ItemPresentation; import com.intellij.navigation.NavigationItemFileStatus; -import com.intellij.openapi.util.NlsSafe; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vcs.FileStatus; import com.intellij.openapi.vfs.VirtualFile; @@ -18,7 +17,7 @@ import javax.swing.*; /** * This class represent a group of elements in the "Find Usages" window. - * @author shahariel + * @author Shahar Ariel */ public class SdUsageGroup implements UsageGroup { private final VirtualFile myFile; @@ -91,4 +90,7 @@ public class SdUsageGroup implements UsageGroup { public @Nullable Icon getIcon() { return myIcon; } + + @Override + public void update() {} // here because JetBrains asked } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java index 417cab307d4..46b4e738fcb 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java @@ -7,11 +7,9 @@ import com.intellij.ide.hierarchy.HierarchyTreeStructure; import com.intellij.ide.util.treeView.NodeDescriptor; import com.intellij.openapi.actionSystem.ActionGroup; import com.intellij.openapi.actionSystem.ActionManager; -import com.intellij.openapi.actionSystem.ActionPlaces; import com.intellij.openapi.actionSystem.IdeActions; import com.intellij.openapi.project.Project; import com.intellij.psi.PsiElement; -import com.intellij.ui.PopupHandler; import com.intellij.util.ObjectUtils; import org.intellij.sdk.language.psi.SdFunctionDefinition; import org.jetbrains.annotations.NotNull; @@ -24,7 +22,7 @@ import javax.swing.JTree; /** * This class is a browser for the "Call Hierarchy" window. - * @author shahariel + * @author Shahar Ariel */ public class SdCallHierarchyBrowser extends CallHierarchyBrowserBase { @@ -46,10 +44,8 @@ public class SdCallHierarchyBrowser extends CallHierarchyBrowserBase { type2TreeMap.put(getCalleeType(), createHierarchyTree(group)); } - private JTree createHierarchyTree(ActionGroup group) { - final JTree tree = createTree(false); - PopupHandler.installPopupMenu(tree, group, ActionPlaces.CALL_HIERARCHY_VIEW_POPUP); - return tree; + private @NotNull JTree createHierarchyTree(ActionGroup group) { + return createTree(false); } @Override diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java index df6db46ff6f..fdd8871201e 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java @@ -22,7 +22,7 @@ import javax.swing.Icon; /** * This class represents a node descriptor to a node in a tree in the "Call Hierarchy" window. - * @author shahariel + * @author Shahar Ariel */ public class SdCallHierarchyNodeDescriptor extends HierarchyNodeDescriptor { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java index 5566e1f71e1..4808c55f1ae 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java @@ -20,7 +20,7 @@ import com.intellij.openapi.project.Project; /** * This class is used for the extension (in plugin.xml), to enable "Call Hierarchy" window using the plugin code. - * @author shahariel + * @author Shahar Ariel */ public class SdCallHierarchyProvider implements HierarchyProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java index a5d8492b521..0f843d9fe6a 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java @@ -21,7 +21,7 @@ import java.util.List; /** * This abstract class represents a general tree in the "Call Hierarchy" window. - * @author shahariel + * @author Shahar Ariel */ public abstract class SdCallTreeStructure extends HierarchyTreeStructure { protected final String myScopeType; diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java index f683fe0139a..cce1ebf172a 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java @@ -18,7 +18,7 @@ import java.util.List; /** * This class represents a Callee tree in the "Call Hierarchy" window. - * @author shahariel + * @author Shahar Ariel */ public class SdCalleeTreeStructure extends SdCallTreeStructure { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java index 4a30dd7492f..1ab2f6613d9 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java @@ -20,7 +20,7 @@ import java.util.function.Consumer; /** * This class represents a Caller tree in the "Call Hierarchy" window. - * @author shahariel + * @author Shahar Ariel */ public class SdCallerTreeStructure extends SdCallTreeStructure { @@ -52,7 +52,7 @@ public class SdCallerTreeStructure extends SdCallTreeStructure { ProgressManager.checkCanceled(); PsiElement psiElement = r.getElement(); SdFunctionDefinition f = PsiTreeUtil.getParentOfType(psiElement, SdFunctionDefinition.class, false); - if (f != null && !f.getName().equals(macroName)) { + if (f != null && f.getName() != null && !f.getName().equals(macroName)) { ContainerUtil.addIfNotNull(results, f); } else { SdFirstPhaseDefinition fp = PsiTreeUtil.getParentOfType(psiElement, SdFirstPhaseDefinition.class, false); diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java index c3e59124ef8..19f9d72f6f4 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java @@ -18,7 +18,7 @@ import java.util.HashSet; /** * This class is util class to the Call Hierarchy feature. - * @author shahariel + * @author Shahar Ariel */ public class SdHierarchyUtil { @@ -42,7 +42,8 @@ public class SdHierarchyUtil { HashSet<SdRankProfileDefinition> tempRanks = new HashSet<>(); SdRankProfileDefinition curRank = rank; while (curRank != null) { - if (curRank.getName().equals(rankProfileTarget.getName())) { + String curRankName = curRank.getName(); + if (curRankName != null && curRankName.equals(rankProfileTarget.getName())) { result.addAll(tempRanks); break; } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex b/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex index e24769c9d2a..47dddb9c33f 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex @@ -12,7 +12,7 @@ import static com.intellij.psi.TokenType.WHITE_SPACE; // Pre-defined whitespace /* * This file is used for the generation of the lexer of the SD language. -* @author shahariel +* @author Shahar Ariel */ %% diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java index b084dad7816..fbaa8d0cac5 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java @@ -20,7 +20,7 @@ import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml), to make the parsing process use the plugin code. - * @author shahariel + * @author Shahar Ariel */ public class SdParserDefinition implements ParserDefinition { public static final TokenSet WHITE_SPACES = TokenSet.create(TokenType.WHITE_SPACE); diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf index 404f8437865..6ae9d26729c 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf @@ -1,7 +1,7 @@ /** This file is the SD grammar. NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many) -@author: shahariel +@author: Shahar Ariel */ { @@ -330,7 +330,7 @@ KeywordOrIdentifier ::= schema | search | document | struct | field | type | ind bolding | on | off | true | false | id | normalizing | stemming | arity | hnsw | dictionary | hash | btree | fieldset | fields | constant | annotation | attribute | body | header | index | - reference | summary | set_language a + reference | summary | set_language // Note- in this form, those keywords can't be use as identifier-with-dash! KeywordNotIdentifier ::= struct-field | document-summary | omit-summary-features | from-disk | rank-profile | rank-type | diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java index 3fa6739161b..4f11a2b4b67 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java @@ -6,6 +6,6 @@ import com.intellij.psi.PsiElement; /** * This interface represents a declaration in the SD language. - * @author shahariel + * @author Shahar Ariel */ public interface SdDeclaration extends PsiElement, NavigationItem, SdNamedElement {} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java index 5ea5968f0bb..c19e21d0f19 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java @@ -3,7 +3,7 @@ package org.intellij.sdk.language.psi; /** * This Enum describes the different declarations' types and their names. - * @author shahariel + * @author Shahar Ariel */ public enum SdDeclarationType { DOCUMENT("Document"), diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java index b2e99a9538a..76c4454b3b2 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java @@ -11,7 +11,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable "find Usages" window take the element description from * here. Used only for the "target" element. - * @author shahariel + * @author Shahar Ariel */ public class SdElementDescriptionProvider implements ElementDescriptionProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java index 9cb9690cd5b..1f3b98b6012 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java @@ -9,7 +9,7 @@ import org.intellij.sdk.language.SdFileType; /** * This class is a factory of psi elements in the SD PSI tree. - * @author shahariel + * @author Shahar Ariel */ public class SdElementFactory { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java index a091d444d8e..b26cd66455d 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java @@ -8,7 +8,7 @@ import org.jetbrains.annotations.NotNull; /** * This class represent a SdElementType. - * @author shahariel + * @author Shahar Ariel */ public class SdElementType extends IElementType { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java index c044bdeeddd..c2d9249bce1 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java @@ -10,7 +10,7 @@ import org.jetbrains.annotations.NotNull; /** * This class represent an SD file. - * @author shahariel + * @author Shahar Ariel */ public class SdFile extends PsiFileBase { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java index d3738171343..78d964b45c9 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java @@ -5,7 +5,7 @@ import org.intellij.sdk.language.SdUtil; /** * This interface represents a function's declaration in the SD language. - * @author shahariel + * @author Shahar Ariel */ public interface SdFunctionDefinitionInterface extends SdDeclaration { default boolean isOverride() { @@ -17,7 +17,7 @@ public interface SdFunctionDefinitionInterface extends SdDeclaration { } while (curRankProfile != null) { for (SdFunctionDefinition macro : PsiTreeUtil.collectElementsOfType(curRankProfile, SdFunctionDefinition.class)) { - if (macro.getName().equals(macroName)) { + if (macro.getName() != null && macro.getName().equals(macroName)) { return true; } } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java index 141d0abd403..e20eb4cb6c4 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java @@ -6,7 +6,7 @@ import com.intellij.psi.util.PsiTreeUtil; /** * This interface represents an identifier in the SD language (regular identifiers and identifiers with dash). - * @author shahariel + * @author Shahar Ariel */ public interface SdIdentifier extends PsiElement { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java index bd74a87b551..6e56f548410 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java @@ -6,7 +6,7 @@ import com.intellij.psi.PsiNameIdentifierOwner; /** * This interface is used to wrap a Psi Element with SdNamedElement interface, which enables the element to be a * "name owner" (like an identifier). It allows the element to take a part in references, find usages and more. - * @author shahariel + * @author Shahar Ariel */ public interface SdNamedElement extends PsiNameIdentifierOwner { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java index 98e7494389d..1024995832d 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java @@ -8,7 +8,7 @@ import org.jetbrains.annotations.NotNull; /** * This class represent a SdTokenType. - * @author shahariel + * @author Shahar Ariel */ public class SdTokenType extends IElementType { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java new file mode 100644 index 00000000000..1a13318a38f --- /dev/null +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java @@ -0,0 +1,58 @@ +package org.intellij.sdk.language.psi.impl; + +import com.intellij.extapi.psi.ASTWrapperPsiElement; +import com.intellij.lang.ASTNode; +import com.intellij.navigation.ItemPresentation; +import com.intellij.psi.util.PsiTreeUtil; +import org.intellij.sdk.language.SdIcons; +import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import javax.swing.Icon; + +/** + * This class is used for methods' implementations for SdFirstPhaseDefinition. Connected with "mixin" to + * FirstPhaseDefinition rule in sd.bnf + * @author Shahar Ariel + */ +public class SdFirstPhaseDefinitionMixin extends ASTWrapperPsiElement { + + public SdFirstPhaseDefinitionMixin(@NotNull ASTNode node) { + super(node); + } + + @NotNull + public String getName() { + SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(this, SdRankProfileDefinition.class); + if (rankProfile == null) { + return ""; + } + return "first-phase of " + rankProfile.getName(); + } + + public ItemPresentation getPresentation() { + final SdFirstPhaseDefinitionMixin element = this; + return new ItemPresentation() { + @Override + public String getPresentableText() { + SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class); + if (rankProfile == null) { + return ""; + } + return "first-phase of " + rankProfile.getName(); + } + + @Nullable + @Override + public String getLocationString() { + return element.getContainingFile() != null ? element.getContainingFile().getName() : null; + } + + @Override + public Icon getIcon(boolean unused) { + return SdIcons.FIRST_PHASE; + } + }; + } +} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java index 951f90b0793..536dfc4f2cc 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java @@ -11,7 +11,7 @@ import org.jetbrains.annotations.NotNull; /** * This abstract class is used for methods' implementations for SdIdentifier. Connected with "mixin" to IdentifierVal and * IdentifierWithDashVal rules in sd.bnf - * @author shahariel + * @author Shahar Ariel */ public abstract class SdIdentifierMixin extends SdIdentifierMixinImpl implements PsiNamedElement { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java index 69ccf9eda2a..7b738cc1b0f 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java @@ -10,7 +10,7 @@ import org.jetbrains.annotations.NotNull; /** * This class is used for methods' implementations for SdIdentifier. The abstract class SdIdentifierMixin extents it. - * @author shahariel + * @author Shahar Ariel */ public class SdIdentifierMixinImpl extends ASTWrapperPsiElement implements SdIdentifier { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java index 0bc1a1eaad3..2c2a7aae3b1 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java @@ -38,7 +38,7 @@ import javax.swing.Icon; /** * This abstract class is used to wrap a Psi Element with SdNamedElement interface, which enables the element to be a * "name owner" (like an identifier). It allows the element to take a part in references, find usages and more. - * @author shahariel + * @author Shahar Ariel */ public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements SdNamedElement { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java index 7c56acce1e9..085b1e04102 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java @@ -13,7 +13,7 @@ import javax.swing.Icon; /** * This class is used for methods' implementations for SdSummaryDefinition. Connected with "mixin" to SummaryDefinition * rule in sd.bnf - * @author shahariel + * @author Shahar Ariel */ public abstract class SdSummaryDefinitionMixin extends ASTWrapperPsiElement { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java index 493047c0363..67bda9ced19 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java @@ -23,7 +23,7 @@ import java.util.List; /** * This class is used for the presentation of an element in the "Structure View" window. - * @author shahariel + * @author Shahar Ariel */ public class SdStructureViewElement implements StructureViewTreeElement, SortableTreeElement { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java index 6ca4c979ef0..2e902beb2ad 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java +++ b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java @@ -13,7 +13,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdStructureViewModel. It make the IDE use our * plugin's code when creating the "Structure View" window. - * @author shahariel + * @author Shahar Ariel */ public class SdStructureViewFactory implements PsiStructureViewFactory { diff --git a/sd-plugin/src/main/resources/META-INF/plugin.xml b/sd-plugin/src/main/resources/META-INF/plugin.xml index 118dbb16ad5..10a617fe3d4 100644 --- a/sd-plugin/src/main/resources/META-INF/plugin.xml +++ b/sd-plugin/src/main/resources/META-INF/plugin.xml @@ -2,10 +2,10 @@ <idea-plugin> <id>org.intellij.sdk.language</id> <name>SdReader</name> - <version>1.0.1</version> + <version>1.0.0</version> <!-- Text to display as company information on Preferences/Settings | Plugin page --> - <vendor>Vespa</vendor> + <vendor>Yahoo</vendor> <!-- Product and plugin compatibility requirements --> <depends>com.intellij.java</depends> diff --git a/sd-plugin/src/main/resources/icons/override_macro_icon.png b/sd-plugin/src/main/resources/icons/override_macro_icon.png Binary files differindex ae9e2636798..024eb777eef 100644 --- a/sd-plugin/src/main/resources/icons/override_macro_icon.png +++ b/sd-plugin/src/main/resources/icons/override_macro_icon.png diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index be1e2c79f70..aa118600272 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -142,8 +142,7 @@ struct IndexManagerTest : public ::testing::Test { void resetIndexManager(); void removeDocument(uint32_t docId, SerialNum serialNum) { runAsIndex([&]() { _index_manager->removeDocument(docId, serialNum); - _index_manager->commit(serialNum, - emptyDestructorCallback); + _index_manager->commit(serialNum, emptyDestructorCallback); }); _writeService.indexFieldWriter().sync_all(); } diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 11898d1a630..e4eaad538cb 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -79,8 +79,8 @@ public: void insertDocument(uint32_t lid, const document::Document &doc) override { _index.insertDocument(lid, doc); } - void removeDocument(uint32_t lid) override { - _index.removeDocument(lid); + void removeDocuments(LidVector lids) override { + _index.removeDocuments(std::move(lids)); } uint64_t getStaticMemoryFootprint() const override { return _index.getStaticMemoryFootprint(); diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index 2602ee377b7..f7702708c53 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -226,13 +226,6 @@ CombiningFeedView::heartBeat(search::SerialNum serialNum) } void -CombiningFeedView::sync() -{ - getReadyFeedView()->sync(); - // Assume this synced all feed views due to sharing of threads. -} - -void CombiningFeedView::forceCommit(const CommitParam & param, DoneCallback onDone) { for (const auto &view : _views) { diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index 9df2c4c0c0f..3827b491896 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -76,7 +76,6 @@ public: void prepareMove(MoveOperation &putOp) override; void handleMove(const MoveOperation &moveOp, std::shared_ptr<vespalib::IDestructorCallback> moveDoneCtx) override; void heartBeat(search::SerialNum serialNum) override; - void sync() override; void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp) override; void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index b8985a83ef1..ce7f1d70195 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -87,11 +87,4 @@ FastAccessFeedView::internalForceCommit(const CommitParam & param, OnForceCommit Parent::internalForceCommit(param, onCommitDone); } -void -FastAccessFeedView::sync() -{ - Parent::sync(); - _writeService.attributeFieldWriter().sync_all(); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h index fae9527920a..a27bf04d701 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h @@ -63,7 +63,6 @@ public: } void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; - void sync() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 55549e986b8..af746f9debb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -300,13 +300,6 @@ FeedHandler::performJoin(FeedToken token, JoinBucketsOperation &op) { } void -FeedHandler::performSync() -{ - assert(_writeService.master().isCurrentThread()); - _activeFeedView->sync(); -} - -void FeedHandler::performEof() { assert(_writeService.master().isCurrentThread()); @@ -769,13 +762,6 @@ FeedHandler::heartBeat() _activeFeedView->heartBeat(_serialNum); } -void -FeedHandler::sync() -{ - _writeService.master().execute(makeLambdaTask([this]() { performSync(); })); - _writeService.sync(); -} - FeedHandler::RPC::Result FeedHandler::receive(const Packet &packet) { diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 026d056b31c..ef15b268086 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -123,7 +123,6 @@ private: void performDeleteBucket(FeedToken token, DeleteBucketOperation &op); void performSplit(FeedToken token, SplitBucketOperation &op); void performJoin(FeedToken token, JoinBucketsOperation &op); - void performSync(); void performEof(); /** @@ -236,7 +235,6 @@ public: void handleMove(MoveOperation &op, std::shared_ptr<vespalib::IDestructorCallback> moveDoneCtx) override; void heartBeat() override; - void sync(); RPC::Result receive(const Packet &packet) override; void eof() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h index 4015589f43a..4feb7d013ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h @@ -59,7 +59,6 @@ public: virtual void prepareMove(MoveOperation &putOp) = 0; virtual void handleMove(const MoveOperation &putOp, DoneCallback onDone) = 0; virtual void heartBeat(search::SerialNum serialNum) = 0; - virtual void sync() = 0; virtual void forceCommit(const CommitParam & param, DoneCallback onDone) = 0; void forceCommit(CommitParam param) { forceCommit(param, DoneCallback()); } void forceCommit(search::SerialNum serialNum) { forceCommit(CommitParam(serialNum)); } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 6101e8f0918..d973777020d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -40,24 +40,6 @@ SearchableFeedView::SearchableFeedView(StoreOnlyFeedView::Context storeOnlyCtx, SearchableFeedView::~SearchableFeedView() = default; void -SearchableFeedView::performSync() -{ - // Called by index write thread, delays when sync() method on it completes. - assert(_writeService.index().isCurrentThread()); - _writeService.indexFieldInverter().sync_all(); - _writeService.indexFieldWriter().sync_all(); -} - -void -SearchableFeedView::sync() -{ - assert(_writeService.master().isCurrentThread()); - Parent::sync(); - _writeService.index().execute(makeLambdaTask([this]() { performSync(); })); - _writeService.index().sync(); -} - -void SearchableFeedView::putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const DocumentSP &newDoc, OnOperationDoneType onWriteDone) { @@ -188,11 +170,13 @@ void SearchableFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) { Parent::handleCompactLidSpace(op); + vespalib::Gate gate; _writeService.index().execute( - makeLambdaTask([this, &op]() { + makeLambdaTask([this, &op, &gate]() { _indexWriter->compactLidSpace(op.getSerialNum(), op.getLidLimit()); + gate.countDown(); })); - _writeService.index().sync(); + gate.await(); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h index 8a4c8d814bf..5ff309446d2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h @@ -42,7 +42,6 @@ private: void performIndexHeartBeat(SerialNum serialNum); void internalDeleteBucket(const DeleteBucketOperation &delOp) override; - void performSync(); void heartBeatIndexedFields(SerialNum serialNum) override; void putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const DocumentSP &newDoc, OnOperationDoneType onWriteDone) override; @@ -60,7 +59,6 @@ public: ~SearchableFeedView() override; const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; } void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; - void sync() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 078017ea301..c9a0892d3c3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -173,12 +173,6 @@ StoreOnlyFeedView::Context::Context(Context &&) noexcept = default; StoreOnlyFeedView::Context::~Context() = default; void -StoreOnlyFeedView::sync() -{ - _writeService.summary().sync(); -} - -void StoreOnlyFeedView::forceCommit(const CommitParam & param, DoneCallback onDone) { internalForceCommit(param, std::make_shared<ForceCommitContext>(_writeService.master(), _metaStore, @@ -775,10 +769,12 @@ StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) internalForceCommit(CommitParam(serialNum), commitContext); } if (useDocumentStore(serialNum)) { - _writeService.summary().execute(makeLambdaTask([this, &op]() { + vespalib::Gate gate; + _writeService.summary().execute(makeLambdaTask([this, &op, &gate]() { _summaryAdapter->compactLidSpace(op.getLidLimit()); + gate.countDown(); })); - _writeService.summary().sync(); + gate.await(); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 6b4dada1313..4e7a62548c2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -236,7 +236,6 @@ public: void prepareMove(MoveOperation &putOp) override; void handleMove(const MoveOperation &putOp, std::shared_ptr<vespalib::IDestructorCallback> doneCtx) override; void heartBeat(search::SerialNum serialNum) override; - void sync() override; void forceCommit(const CommitParam & param, DoneCallback onDone) override; /** diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h index e960aa601cb..af88f7fa6b7 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h @@ -29,7 +29,6 @@ struct DummyFeedView : public IFeedView void prepareMove(MoveOperation &) override {} void handleMove(const MoveOperation &, std::shared_ptr<vespalib::IDestructorCallback>) override {} void heartBeat(search::SerialNum) override {} - void sync() override {} void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &) override {} void handleCompactLidSpace(const CompactLidSpaceOperation &) override {} void forceCommit(const CommitParam &, DoneCallback) override { } diff --git a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h index 735beb620f6..bff929206a0 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h +++ b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h @@ -16,6 +16,7 @@ namespace searchcorespi::index { * Interface for a memory index as seen from an index maintainer. */ struct IMemoryIndex : public searchcorespi::IndexSearchable { + using LidVector = std::vector<uint32_t>; using SP = std::shared_ptr<IMemoryIndex>; using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; virtual ~IMemoryIndex() {} @@ -49,7 +50,12 @@ struct IMemoryIndex : public searchcorespi::IndexSearchable { * * @param lid the local document id. */ - virtual void removeDocument(uint32_t lid) = 0; + void removeDocument(uint32_t lid) { + LidVector lids; + lids.push_back(lid); + removeDocuments(std::move(lids)); + } + virtual void removeDocuments(LidVector lids) = 0; /** * Commits the inserts and removes since the last commit, making them searchable. diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 07676b4b330..74848e93411 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -1188,12 +1188,12 @@ IndexMaintainer::removeDocuments(LidVector lids, SerialNum serialNum) assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); for (uint32_t lid : lids) { - _current_index->removeDocument(lid); _selector->setSource(lid, _current_index_id); _source_list->setSource(lid); } _source_selector_changes += lids.size(); _current_serial_num = serialNum; + _current_index->removeDocuments(std::move(lids)); } void diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 9ee47cc6849..dd445745f18 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -227,7 +227,9 @@ struct Fixture { index.insertDocument(id, *doc); } void remove(uint32_t id) { - index.removeDocument(id); + std::vector<uint32_t> lids; + lids.push_back(id); + index.removeDocuments(std::move(lids)); } void readWork(uint32_t cnt); diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 59ade118990..20cfb045081 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -112,7 +112,9 @@ struct Index { return d; } Index &remove(uint32_t id) { - index.removeDocument(id); + std::vector<uint32_t> lids; + lids.push_back(id); + index.removeDocuments(std::move(lids)); internalSyncCommit(); return *this; } diff --git a/searchlib/src/vespa/searchlib/common/unique_issues.h b/searchlib/src/vespa/searchlib/common/unique_issues.h index 79cb7a9aa23..17c42da060c 100644 --- a/searchlib/src/vespa/searchlib/common/unique_issues.h +++ b/searchlib/src/vespa/searchlib/common/unique_issues.h @@ -18,7 +18,8 @@ private: public: using UP = std::unique_ptr<UniqueIssues>; void handle(const vespalib::Issue &issue) override; - void for_each_message(auto fun) const { + template <class Function> + void for_each_message(Function fun) const { for (const auto &msg: _messages) { fun(msg); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index 4f03a5cb95f..127ff1d52c3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -116,6 +116,7 @@ DocumentInverter::buildFieldPath(const document::DocumentType &docType, void DocumentInverter::invertDocument(uint32_t docId, const Document &doc) { + // Might want to batch inverters as we do for attributes const document::DataType *dataType(doc.getDataType()); if (_indexedFieldPaths.empty() || _dataType != dataType) { buildFieldPath(doc.getType(), dataType); @@ -152,20 +153,31 @@ DocumentInverter::invertDocument(uint32_t docId, const Document &doc) } void -DocumentInverter::removeDocument(uint32_t docId) +DocumentInverter::removeDocument(uint32_t docId) { + LidVector lids; + lids.push_back(docId); + removeDocuments(std::move(lids)); +} +void +DocumentInverter::removeDocuments(LidVector lids) { + // Might want to batch inverters as we do for attributes for (uint32_t fieldId : _schemaIndexFields._textFields) { FieldInverter *inverter = _inverters[fieldId].get(); - _invertThreads.execute(fieldId, [inverter, docId]() { - inverter->removeDocument(docId); + _invertThreads.execute(fieldId, [inverter, lids]() { + for (uint32_t lid : lids) { + inverter->removeDocument(lid); + } }); } uint32_t urlId = 0; for (const auto & fi : _schemaIndexFields._uriFields) { uint32_t fieldId = fi._all; UrlFieldInverter *inverter = _urlInverters[urlId].get(); - _invertThreads.execute(fieldId,[inverter, docId]() { - inverter->removeDocument(docId); + _invertThreads.execute(fieldId, [inverter, lids]() { + for (uint32_t lid : lids) { + inverter->removeDocument(lid); + } }); ++urlId; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index 7ab80e1a2e5..ed06a0b39cc 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -40,6 +40,7 @@ private: void addFieldPath(const document::DocumentType &docType, uint32_t fieldId); void buildFieldPath(const document::DocumentType & docType, const document::DataType *dataType); + using LidVector = std::vector<uint32_t>; using FieldPath = document::Field; using IndexedFieldPaths = std::vector<std::unique_ptr<FieldPath>>; IndexedFieldPaths _indexedFieldPaths; @@ -100,6 +101,7 @@ public: * (using a field inverter) is added to the 'invert threads' executor', then this function returns. */ void removeDocument(uint32_t docId); + void removeDocuments(LidVector lids); FieldInverter *getInverter(uint32_t fieldId) const { return _inverters[fieldId].get(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 177d8e612bd..cb198f4d33a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -93,17 +93,20 @@ MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc) } void -MemoryIndex::removeDocument(uint32_t docId) +MemoryIndex::removeDocuments(LidVector lids) { if (_frozen) { - LOG(warning, "Memory index frozen: ignoring remove of document (%u)", docId); + LOG(warning, "Memory index frozen: ignoring remove of %lu documents", lids.size()); return; } - _inverter->removeDocument(docId); - if (_indexedDocs.find(docId) != _indexedDocs.end()) { - _indexedDocs.erase(docId); - decNumDocs(); + for (uint32_t lid : lids) { + + if (_indexedDocs.find(lid) != _indexedDocs.end()) { + _indexedDocs.erase(lid); + decNumDocs(); + } } + _inverter->removeDocuments(std::move(lids)); } void diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index 83270aaf2ce..c02e66f790a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -41,6 +41,7 @@ class FieldIndexCollection; class MemoryIndex : public queryeval::Searchable { private: using ISequencedTaskExecutor = vespalib::ISequencedTaskExecutor; + using LidVector = std::vector<uint32_t>; index::Schema _schema; ISequencedTaskExecutor &_invertThreads; ISequencedTaskExecutor &_pushThreads; @@ -115,7 +116,7 @@ public: * * This function is async. commit() must be called for changes to take effect. */ - void removeDocument(uint32_t docId); + void removeDocuments(LidVector lids); /** * Commits the inserts and removes since the last commit, making them searchable. diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index 4daec4c0689..5e66b364242 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -21,6 +21,7 @@ #include <vespa/storage/common/cluster_context.h> #include <vespa/storage/common/messagesender.h> #include <vespa/vespalib/util/monitored_refcount.h> +#include <atomic> namespace vespalib { class ISequencedTaskExecutor; } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 20121d8d193..29fd419daa2 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -68,6 +68,7 @@ import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.Exceptions.RunnableThrowingIOException; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -83,9 +84,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Queue; import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executors; import java.util.concurrent.Phaser; import java.util.concurrent.ScheduledExecutorService; @@ -577,12 +580,19 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private static class JsonResponse implements AutoCloseable { private static final ByteBuffer emptyBuffer = ByteBuffer.wrap(new byte[0]); + private static final int FLUSH_SIZE = 128; private final BufferedContentChannel buffer = new BufferedContentChannel(); private final OutputStream out = new ContentChannelOutputStream(buffer); private final JsonGenerator json; private final ResponseHandler handler; + private final Queue<CompletionHandler> acks = new ConcurrentLinkedQueue<>(); + private final Queue<ByteArrayOutputStream> docs = new ConcurrentLinkedQueue<>(); + private final AtomicLong documentsWritten = new AtomicLong(); + private final AtomicLong documentsFlushed = new AtomicLong(); + private final AtomicLong documentsAcked = new AtomicLong(); private boolean documentsDone = false; + private boolean first = true; private ContentChannel channel; private JsonResponse(ResponseHandler handler) throws IOException { @@ -695,19 +705,72 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { json.writeArrayFieldStart("documents"); } - synchronized void writeDocumentValue(Document document, CompletionHandler completionHandler) { - if ( ! documentsDone) - new JsonWriter(json).write(document); - + /** Writes documents to an internal queue, which is flushed regularly. */ + void writeDocumentValue(Document document, CompletionHandler completionHandler) throws IOException { if (completionHandler != null) { - if ( ! documentsDone) - buffer.write(emptyBuffer, completionHandler); + acks.add(completionHandler); + ackDocuments(); + } + + // Serialise document and add to queue, not necessarily in the order dictated by "written" above, + // i.e., the first 128 documents in the queue are not necessarily the ones ack'ed early. + ByteArrayOutputStream myOut = new ByteArrayOutputStream(1); + myOut.write(','); // Prepend rather than append, to avoid double memory copying. + try (JsonGenerator myJson = jsonFactory.createGenerator(myOut)) { + new JsonWriter(myJson).write(document); + } + docs.add(myOut); + + // Flush the first FLUSH_SIZE documents in the queue to the network layer if chunk is filled. + if (documentsWritten.incrementAndGet() % FLUSH_SIZE == 0) { + flushDocuments(); + } + } + + void ackDocuments() { + while (documentsAcked.incrementAndGet() <= documentsFlushed.get() + FLUSH_SIZE) { + CompletionHandler ack = acks.poll(); + if (ack != null) + ack.completed(); else - completionHandler.completed(); + break; } + documentsAcked.decrementAndGet(); // We overshoot by one above, so decrement again when done. + } + + synchronized void flushDocuments() throws IOException { + for (int i = 0; i < FLUSH_SIZE; i++) { + ByteArrayOutputStream doc = docs.poll(); + if (doc == null) + break; + + if ( ! documentsDone) { + if (first) { // First chunk, remove leading comma from first document, and flush "json" to "buffer". + json.flush(); + buffer.write(ByteBuffer.wrap(doc.toByteArray(), 1, doc.size() - 1), null); + first = false; + } + else { + buffer.write(ByteBuffer.wrap(doc.toByteArray()), null); + } + } + } + + // Ensure new, eligible acks are done, after flushing these documents. + buffer.write(emptyBuffer, new CompletionHandler() { + @Override public void completed() { + documentsFlushed.addAndGet(FLUSH_SIZE); + ackDocuments(); + } + @Override public void failed(Throwable t) { + log.log(WARNING, "Error writing documents", t); + completed(); + } + }); } synchronized void writeArrayEnd() throws IOException { + flushDocuments(); documentsDone = true; json.writeEndArray(); } @@ -989,9 +1052,6 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { if (cluster.isEmpty() && path.documentType().isEmpty()) throw new IllegalArgumentException("Must set 'cluster' parameter to a valid content cluster id when visiting at a root /document/v1/ level"); - Optional<Integer> slices = getProperty(request, SLICES, integerParser); - Optional<Integer> sliceId = getProperty(request, SLICE_ID, integerParser); - VisitorParameters parameters = parseCommonParameters(request, path, cluster); parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(AllFields.NAME))); parameters.setMaxTotalHits(wantedDocumentCount); @@ -1006,10 +1066,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { parameters.setThrottlePolicy(throttlePolicy); parameters.visitInconsistentBuckets(true); parameters.setSessionTimeoutMs(Math.max(1, request.getTimeout(TimeUnit.MILLISECONDS) - handlerTimeout.toMillis())); - if (slices.isPresent() && sliceId.isPresent()) - parameters.slice(slices.get(), sliceId.get()); - else if (slices.isPresent() != sliceId.isPresent()) - throw new IllegalArgumentException("None or both of '" + SLICES + "' and '" + SLICE_ID + "' must be set"); + return parameters; } @@ -1044,6 +1101,13 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { List.of(FixedBucketSpaces.defaultSpace(), FixedBucketSpaces.globalSpace()), getProperty(request, BUCKET_SPACE))); + Optional<Integer> slices = getProperty(request, SLICES, integerParser); + Optional<Integer> sliceId = getProperty(request, SLICE_ID, integerParser); + if (slices.isPresent() && sliceId.isPresent()) + parameters.slice(slices.get(), sliceId.get()); + else if (slices.isPresent() != sliceId.isPresent()) + throw new IllegalArgumentException("None or both of '" + SLICES + "' and '" + SLICE_ID + "' must be set"); + return parameters; } @@ -1124,14 +1188,22 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { response.writeDocumentsArrayStart(); } @Override public void onDocument(JsonResponse response, Document document, Runnable ack, Consumer<String> onError) { - if (streamed) - response.writeDocumentValue(document, new CompletionHandler() { - @Override public void completed() { ack.run();} - @Override public void failed(Throwable t) { ack.run(); onError.accept(t.getMessage()); } - }); - else { - response.writeDocumentValue(document, null); - ack.run(); + try { + if (streamed) + response.writeDocumentValue(document, new CompletionHandler() { + @Override public void completed() { ack.run();} + @Override public void failed(Throwable t) { + ack.run(); + onError.accept(t.getMessage()); + } + }); + else { + response.writeDocumentValue(document, null); + ack.run(); + } + } + catch (Exception e) { + onError.accept(e.getMessage()); } } @Override public void onEnd(JsonResponse response) throws IOException { diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index e61dc071b62..688d98ff032 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/backtrace.h> #include <vespa/vespalib/util/size_literals.h> +#include <atomic> #include <thread> using namespace vespalib; |
