diff options
70 files changed, 463 insertions, 1629 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index f30074383ee..e97f601fb74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,7 @@ add_subdirectory(searchcore) add_subdirectory(searchcorespi) add_subdirectory(searchlib) add_subdirectory(searchsummary) +add_subdirectory(security-utils) add_subdirectory(service-monitor) add_subdirectory(simplemetrics) add_subdirectory(slobrok) diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index 16accb368fd..9443339624a 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -101,18 +101,17 @@ class OverrideProcessor implements PreProcessor { retainMostSpecificEnvironmentAndRegion(parent, children, context); } - /** - * Ensures that environment and region does not change from something non-default to something else. - */ private void checkConsistentInheritance(List<Element> children, Context context) { for (Element child : children) { Set<Environment> environments = getEnvironments(child); Set<RegionName> regions = getRegions(child); - if ( ! environments.isEmpty() && ! context.environments.isEmpty() && !environments.equals(context.environments)) { - throw new IllegalArgumentException("Environments in child (" + environments + ") differs from that inherited from parent (" + context.environments + ") at " + child); + if ( ! environments.isEmpty() && ! context.environments.isEmpty() && ! context.environments.containsAll(environments)) { + throw new IllegalArgumentException("Environments in child (" + environments + + ") are not a subset of those of the parent (" + context.environments + ") at " + child); } - if ( ! regions.isEmpty() && ! context.regions.isEmpty() && ! regions.equals(context.regions)) { - throw new IllegalArgumentException("Regions in child (" + regions + ") differs from that inherited from parent (" + context.regions + ") at " + child); + if ( ! regions.isEmpty() && ! context.regions.isEmpty() && ! context.regions.containsAll(regions)) { + throw new IllegalArgumentException("Regions in child (" + regions + + ") are not a subset of those of the parent (" + context.regions + ") at " + child); } } } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java index d01faaaaea9..4c6a3eb3513 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java @@ -38,7 +38,7 @@ public class HostedOverrideProcessorTest { @Test - public void testParsingDefault() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException { + public void testParsingDefault() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -50,7 +50,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingEnvironmentAndRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentAndRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -62,7 +62,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingEnvironmentAndRegion2() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentAndRegion2() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -74,7 +74,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingEnvironmentAndRegion3() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentAndRegion3() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -86,7 +86,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentUnknownRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -98,7 +98,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingEnvironmentNoRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentNoRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -110,7 +110,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingUnknownEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingUnknownEnvironment() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -122,7 +122,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingUnknownEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingUnknownEnvironmentUnknownRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -134,7 +134,7 @@ public class HostedOverrideProcessorTest { } @Test - public void testParsingInheritEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingInheritEnvironment() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + diff --git a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java index 32a1c4f847f..62e6671120b 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java @@ -34,7 +34,7 @@ public class OverrideProcessorTest { " </admin>" + " <content id=\"foo\" version=\"1.0\">" + " <redundancy>1</redundancy>" + - " <documents>" + + " <documents deploy:environment='staging prod'>" + " <document mode='index' type='music'/>\n" + " <document type='music2' mode='index' />\n" + " <document deploy:environment='prod' deploy:region='us-east-3' mode='index' type='music'/>\n" + @@ -42,6 +42,10 @@ public class OverrideProcessorTest { " <document deploy:environment='prod' mode='index' type='music3'/>\n" + " <document deploy:environment='prod' deploy:region='us-west' mode='index' type='music4'/>\n" + " </documents>" + + " <documents>" + + " <document mode='store-only' type='music'/>\n" + + " <document type='music2' mode='streaming' />\n" + + " </documents>" + " <nodes>" + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + " </nodes>" + @@ -73,7 +77,7 @@ public class OverrideProcessorTest { @Test - public void testParsingDefault() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException { + public void testParsingDefault() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + " <admin version=\"2.0\">" + @@ -82,8 +86,8 @@ public class OverrideProcessorTest { " <content id=\"foo\" version=\"1.0\">" + " <redundancy>1</redundancy>" + " <documents>" + - " <document mode=\"index\" type=\"music\"/>" + - " <document mode=\"index\" type=\"music2\"/>" + + " <document mode=\"store-only\" type=\"music\"/>" + + " <document mode=\"streaming\" type=\"music2\"/>" + " </documents>" + " <nodes>" + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + @@ -101,7 +105,7 @@ public class OverrideProcessorTest { } @Test - public void testParsingEnvironmentAndRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentAndRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -133,7 +137,7 @@ public class OverrideProcessorTest { } @Test - public void testParsingEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentUnknownRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -143,7 +147,9 @@ public class OverrideProcessorTest { " <content id=\"foo\" version=\"1.0\">" + " <redundancy>1</redundancy>" + " <documents>" + - " <document mode=\"index\" type=\"music3\"/>" + + " <document mode='index' type='music'/>\n" + + " <document type='music2' mode='index' />\n" + + " <document mode='index' type='music3'/>" + " </documents>" + " <nodes>" + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + @@ -164,7 +170,7 @@ public class OverrideProcessorTest { } @Test - public void testParsingEnvironmentNoRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingEnvironmentNoRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -174,7 +180,9 @@ public class OverrideProcessorTest { " <content id=\"foo\" version=\"1.0\">" + " <redundancy>1</redundancy>" + " <documents>" + - " <document mode=\"index\" type=\"music3\"/>" + + " <document mode='index' type='music'/>\n" + + " <document type='music2' mode='index' />\n" + + " <document mode='index' type='music3'/>" + " </documents>" + " <nodes>" + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + @@ -195,7 +203,7 @@ public class OverrideProcessorTest { } @Test - public void testParsingUnknownEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingDevEnvironment() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -205,8 +213,8 @@ public class OverrideProcessorTest { " <content id=\"foo\" version=\"1.0\">" + " <redundancy>1</redundancy>" + " <documents>" + - " <document mode=\"index\" type=\"music\"/>" + - " <document mode=\"index\" type=\"music2\"/>" + + " <document mode=\"store-only\" type=\"music\"/>" + + " <document mode=\"streaming\" type=\"music2\"/>" + " </documents>" + " <nodes>" + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + @@ -224,7 +232,7 @@ public class OverrideProcessorTest { } @Test - public void testParsingUnknownEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingTestEnvironmentUnknownRegion() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -234,8 +242,8 @@ public class OverrideProcessorTest { " <content id=\"foo\" version=\"1.0\">" + " <redundancy>1</redundancy>" + " <documents>" + - " <document mode=\"index\" type=\"music\"/>" + - " <document mode=\"index\" type=\"music2\"/>" + + " <document mode=\"store-only\" type=\"music\"/>" + + " <document mode=\"streaming\" type=\"music2\"/>" + " </documents>" + " <nodes>" + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + @@ -253,7 +261,7 @@ public class OverrideProcessorTest { } @Test - public void testParsingInheritEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingInheritEnvironment() throws TransformerException { String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + @@ -263,8 +271,8 @@ public class OverrideProcessorTest { " <content id=\"foo\" version=\"1.0\">" + " <redundancy>1</redundancy>" + " <documents>" + - " <document mode=\"index\" type=\"music\"/>" + - " <document mode=\"index\" type=\"music2\"/>" + + " <document mode='index' type='music'/>\n" + + " <document type='music2' mode='index' />\n" + " </documents>" + " <nodes>" + // node1 is specified for us-west but does not match because region overrides implies environment=prod @@ -284,7 +292,7 @@ public class OverrideProcessorTest { } @Test(expected = IllegalArgumentException.class) - public void testParsingDifferentEnvInParentAndChild() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingDifferentEnvInParentAndChild() throws TransformerException { String in = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + " <admin deploy:environment=\"prod\" version=\"2.0\">" + @@ -296,7 +304,7 @@ public class OverrideProcessorTest { } @Test(expected = IllegalArgumentException.class) - public void testParsingDifferentRegionInParentAndChild() throws ParserConfigurationException, IOException, SAXException, TransformerException { + public void testParsingDifferentRegionInParentAndChild() throws TransformerException { String in = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + " <admin deploy:region=\"us-west\" version=\"2.0\">" + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java index 076161a8584..1994b1096ce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; @@ -26,20 +27,30 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor { public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; - // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do - // not start using the static stuff on SDDocumentTypes any more. + // TODO also traverse struct types to search for bad map or wset types. + // Do this after document manager is fixed, do not start using the static stuff on SDDocumentTypes any more. for (SDField field : search.allConcreteFields()) { - if (field.getDataType() instanceof WeightedSetDataType) { - DataType nestedType = ((WeightedSetDataType)field.getDataType()).getNestedType(); - if ( ! (nestedType instanceof PrimitiveDataType)) { - fail(search, field, "Weighted set must have a primitive key type."); - } - } else if (field.getDataType() instanceof MapDataType) { - if ( ! (((MapDataType)field.getDataType()).getKeyType() instanceof PrimitiveDataType)) { - fail(search, field, "Map key type must be a primitive type"); - } + checkFieldType(field, field.getDataType()); + } + } + + private void checkFieldType(SDField field, DataType dataType) { + if (dataType instanceof ArrayDataType) { + DataType nestedType = ((ArrayDataType) dataType).getNestedType(); + checkFieldType(field, nestedType); + } else if (dataType instanceof WeightedSetDataType) { + DataType nestedType = ((WeightedSetDataType) dataType).getNestedType(); + if ( ! (nestedType instanceof PrimitiveDataType)) { + fail(search, field, "Weighted set must have a primitive key type."); } + } else if (dataType instanceof MapDataType) { + DataType keyType = ((MapDataType) dataType).getKeyType(); + if ( ! (keyType instanceof PrimitiveDataType)) { + fail(search, field, "Map key type must be a primitive type."); + } + checkFieldType(field, ((MapDataType) dataType).getValueType()); } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java index 983f903310f..62b2f2ccbc6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java @@ -28,20 +28,6 @@ public class HttpProvider extends Provider implements ProviderConfig.Producer, @SuppressWarnings("deprecation") private final com.yahoo.search.searchchain.model.federation.HttpProviderSpec providerSpec; - //TODO: For backward compatibility only, eliminate this later - private BinaryScaledAmount cacheSize; - - public double getCacheWeight() { - return providerSpec.cacheWeight; - } - - /** - * TODO: remove, for backward compatibility only. - */ - public void setCacheSize(BinaryScaledAmount cacheSize) { - this.cacheSize = cacheSize; - } - /* * Config producer for the contained http searcher.. */ @@ -101,9 +87,7 @@ public class HttpProvider extends Provider implements ProviderConfig.Producer, } public int cacheSizeMB() { - return providerSpec.cacheSizeMB != null ? - providerSpec.cacheSizeMB : - (int) cacheSize.as(BinaryPrefix.mega); + return providerSpec.cacheSizeMB != null ? providerSpec.cacheSizeMB : 0; } @Override diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index 6b01934307a..aaaebccecfd 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -212,11 +212,11 @@ datatype[24].structtype[0].field[27].detailedtype "" datatype[24].structtype[0].field[28].name "summaryfeatures" datatype[24].structtype[0].field[28].datatype 2 datatype[24].structtype[0].field[28].detailedtype "" -datatype[25].id 171503364 -datatype[25].maptype[0].keytype 1707615575 -datatype[25].maptype[0].valtype 0 -datatype[26].id 1100964733 -datatype[26].arraytype[0].datatype 171503364 +datatype[25].id -372512406 +datatype[25].maptype[0].keytype 0 +datatype[25].maptype[0].valtype 1707615575 +datatype[26].id 1416345047 +datatype[26].arraytype[0].datatype -372512406 datatype[27].id 348447225 datatype[27].structtype[0].name "types.body" datatype[27].structtype[0].version 0 @@ -225,7 +225,7 @@ datatype[27].structtype[0].compresslevel 0 datatype[27].structtype[0].compressthreshold 95 datatype[27].structtype[0].compressminsize 800 datatype[27].structtype[0].field[0].name "complexarray" -datatype[27].structtype[0].field[0].datatype 1100964733 +datatype[27].structtype[0].field[0].datatype 1416345047 datatype[27].structtype[0].field[0].detailedtype "" datatype[28].id -853072901 datatype[28].documenttype[0].name "types" diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index dc7962adec7..97785618e8e 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -560,11 +560,11 @@ documenttype[0].datatype[23].sstruct.field[28].id 1840337115 documenttype[0].datatype[23].sstruct.field[28].id_v6 1981648971 documenttype[0].datatype[23].sstruct.field[28].datatype 2 documenttype[0].datatype[23].sstruct.field[28].detailedtype "" -documenttype[0].datatype[24].id 171503364 +documenttype[0].datatype[24].id -372512406 documenttype[0].datatype[24].type MAP documenttype[0].datatype[24].array.element.id 0 -documenttype[0].datatype[24].map.key.id 1707615575 -documenttype[0].datatype[24].map.value.id 0 +documenttype[0].datatype[24].map.key.id 0 +documenttype[0].datatype[24].map.value.id 1707615575 documenttype[0].datatype[24].wset.key.id 0 documenttype[0].datatype[24].wset.createifnonexistent false documenttype[0].datatype[24].wset.removeifzero false @@ -575,9 +575,9 @@ documenttype[0].datatype[24].sstruct.compression.type NONE documenttype[0].datatype[24].sstruct.compression.level 0 documenttype[0].datatype[24].sstruct.compression.threshold 95 documenttype[0].datatype[24].sstruct.compression.minsize 200 -documenttype[0].datatype[25].id 1100964733 +documenttype[0].datatype[25].id 1416345047 documenttype[0].datatype[25].type ARRAY -documenttype[0].datatype[25].array.element.id 171503364 +documenttype[0].datatype[25].array.element.id -372512406 documenttype[0].datatype[25].map.key.id 0 documenttype[0].datatype[25].map.value.id 0 documenttype[0].datatype[25].wset.key.id 0 @@ -606,9 +606,9 @@ documenttype[0].datatype[26].sstruct.compression.level 0 documenttype[0].datatype[26].sstruct.compression.threshold 95 documenttype[0].datatype[26].sstruct.compression.minsize 200 documenttype[0].datatype[26].sstruct.field[0].name "complexarray" -documenttype[0].datatype[26].sstruct.field[0].id 1028383787 +documenttype[0].datatype[26].sstruct.field[0].id 795629533 documenttype[0].datatype[26].sstruct.field[0].id_v6 658530305 -documenttype[0].datatype[26].sstruct.field[0].datatype 1100964733 +documenttype[0].datatype[26].sstruct.field[0].datatype 1416345047 documenttype[0].datatype[26].sstruct.field[0].detailedtype "" documenttype[0].fieldsets{[document]}.fields[0] "Folders" documenttype[0].fieldsets{[document]}.fields[1] "abyte" diff --git a/config-model/src/test/configmodel/types/types.sd b/config-model/src/test/configmodel/types/types.sd index c55c5611bc5..f34a6776b11 100644 --- a/config-model/src/test/configmodel/types/types.sd +++ b/config-model/src/test/configmodel/types/types.sd @@ -92,7 +92,7 @@ search types { field arrarr type array<array<array<string>>> {header} field maparr type array<map<string, string>> {header} - field complexarray type array< map<array<array<string>>, int> > {body} + field complexarray type array< map<int, array<array<string>>> > {body} struct mystruct { field bytearr type array<byte>{} diff --git a/config-model/src/test/derived/complex/complex.sd b/config-model/src/test/derived/complex/complex.sd index 8fbfd216c3e..82aca00ebfc 100644 --- a/config-model/src/test/derived/complex/complex.sd +++ b/config-model/src/test/derived/complex/complex.sd @@ -41,7 +41,6 @@ search complex { field prefixenabled type string { indexing: index | attribute index: prefix - attribute: prefetch attribute: huge normalizing: none stemming: shortest @@ -58,12 +57,10 @@ search complex { field fleeting type array<float> { indexing: index | attribute - attribute: prefetch } field fleeting2 type float { indexing: index | attribute - attribute: prefetch } field foundat type long { diff --git a/config-model/src/test/derived/complex/summary.cfg b/config-model/src/test/derived/complex/summary.cfg index ff14dbba419..5bb472b5f41 100644 --- a/config-model/src/test/derived/complex/summary.cfg +++ b/config-model/src/test/derived/complex/summary.cfg @@ -1,43 +1,41 @@ defaultsummaryid 1506848752 -classes[].id 1506848752 -classes[].name "default" -classes[].fields[].name "woe" -classes[].fields[].type "longstring" -classes[].fields[].name "exact" -classes[].fields[].type "longstring" -classes[].fields[].name "title" -classes[].fields[].type "longstring" -classes[].fields[].name "dyntitle" -classes[].fields[].type "longstring" -classes[].fields[].name "source" -classes[].fields[].type "longstring" -classes[].fields[].name "stringfield" -classes[].fields[].type "longstring" -classes[].fields[].name "rankfeatures" -classes[].fields[].type "featuredata" -classes[].fields[].name "summaryfeatures" -classes[].fields[].type "featuredata" -classes[].fields[].name "documentid" -classes[].fields[].type "longstring" -classes[].id 28214929 -classes[].name "attributeprefetch" -classes[].fields[].name "year_sub" -classes[].fields[].type "integer" -classes[].fields[].name "prefixenabled" -classes[].fields[].type "longstring" -classes[].fields[].name "fleeting" -classes[].fields[].type "jsonstring" -classes[].fields[].name "fleeting2" -classes[].fields[].type "float" -classes[].fields[].name "foundat" -classes[].fields[].type "int64" -classes[].fields[].name "collapseby" -classes[].fields[].type "integer" -classes[].fields[].name "ts" -classes[].fields[].type "int64" -classes[].fields[].name "combineda" -classes[].fields[].type "integer" -classes[].fields[].name "rankfeatures" -classes[].fields[].type "featuredata" -classes[].fields[].name "summaryfeatures" -classes[].fields[].type "featuredata" +classes[0].id 1506848752 +classes[0].name "default" +classes[0].fields[0].name "woe" +classes[0].fields[0].type "longstring" +classes[0].fields[1].name "exact" +classes[0].fields[1].type "longstring" +classes[0].fields[2].name "title" +classes[0].fields[2].type "longstring" +classes[0].fields[3].name "dyntitle" +classes[0].fields[3].type "longstring" +classes[0].fields[4].name "source" +classes[0].fields[4].type "longstring" +classes[0].fields[5].name "stringfield" +classes[0].fields[5].type "longstring" +classes[0].fields[6].name "rankfeatures" +classes[0].fields[6].type "featuredata" +classes[0].fields[7].name "summaryfeatures" +classes[0].fields[7].type "featuredata" +classes[0].fields[8].name "documentid" +classes[0].fields[8].type "longstring" +classes[1].id 128090024 +classes[1].name "attributeprefetch" +classes[1].fields[0].name "year_sub" +classes[1].fields[0].type "integer" +classes[1].fields[1].name "prefixenabled" +classes[1].fields[1].type "longstring" +classes[1].fields[2].name "fleeting2" +classes[1].fields[2].type "float" +classes[1].fields[3].name "foundat" +classes[1].fields[3].type "int64" +classes[1].fields[4].name "collapseby" +classes[1].fields[4].type "integer" +classes[1].fields[5].name "ts" +classes[1].fields[5].type "int64" +classes[1].fields[6].name "combineda" +classes[1].fields[6].type "integer" +classes[1].fields[7].name "rankfeatures" +classes[1].fields[7].type "featuredata" +classes[1].fields[8].name "summaryfeatures" +classes[1].fields[8].type "featuredata"
\ No newline at end of file diff --git a/config-model/src/test/derived/complex/summarymap.cfg b/config-model/src/test/derived/complex/summarymap.cfg index 1d4b57a49a7..f7ad249ae86 100644 --- a/config-model/src/test/derived/complex/summarymap.cfg +++ b/config-model/src/test/derived/complex/summarymap.cfg @@ -1,34 +1,31 @@ defaultoutputclass -1 -override[].field "dyntitle" -override[].command "dynamicteaser" -override[].arguments "dyntitle" -override[].field "rankfeatures" -override[].command "rankfeatures" -override[].arguments "" -override[].field "summaryfeatures" -override[].command "summaryfeatures" -override[].arguments "" -override[].field "year_sub" -override[].command "attribute" -override[].arguments "year_sub" -override[].field "prefixenabled" -override[].command "attribute" -override[].arguments "prefixenabled" -override[].field "fleeting" -override[].command "attribute" -override[].arguments "fleeting" -override[].field "fleeting2" -override[].command "attribute" -override[].arguments "fleeting2" -override[].field "foundat" -override[].command "attribute" -override[].arguments "foundat" -override[].field "collapseby" -override[].command "attribute" -override[].arguments "collapseby" -override[].field "ts" -override[].command "attribute" -override[].arguments "ts" -override[].field "combineda" -override[].command "attribute" -override[].arguments "combineda" +override[0].field "dyntitle" +override[0].command "dynamicteaser" +override[0].arguments "dyntitle" +override[1].field "rankfeatures" +override[1].command "rankfeatures" +override[1].arguments "" +override[2].field "summaryfeatures" +override[2].command "summaryfeatures" +override[2].arguments "" +override[3].field "year_sub" +override[3].command "attribute" +override[3].arguments "year_sub" +override[4].field "prefixenabled" +override[4].command "attribute" +override[4].arguments "prefixenabled" +override[5].field "fleeting2" +override[5].command "attribute" +override[5].arguments "fleeting2" +override[6].field "foundat" +override[6].command "attribute" +override[6].arguments "foundat" +override[7].field "collapseby" +override[7].command "attribute" +override[7].arguments "collapseby" +override[8].field "ts" +override[8].command "attribute" +override[8].arguments "ts" +override[9].field "combineda" +override[9].command "attribute" +override[9].arguments "combineda"
\ No newline at end of file diff --git a/config-model/src/test/derived/mlr/mlr.sd b/config-model/src/test/derived/mlr/mlr.sd index a6179f836f2..1a5354e3b30 100644 --- a/config-model/src/test/derived/mlr/mlr.sd +++ b/config-model/src/test/derived/mlr/mlr.sd @@ -5,7 +5,6 @@ search mlr { field a type string { indexing: index | summary | attribute - attribute: prefetch } field b type string { @@ -14,7 +13,6 @@ search mlr { field ranklog type string { indexing: attribute - attribute: prefetch } } } diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index c490db59736..647c26e1316 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -209,11 +209,11 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "summaryfeatures" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id 171503364 -datatype[].maptype[].keytype 1707615575 -datatype[].maptype[].valtype 0 -datatype[].id 1100964733 -datatype[].arraytype[].datatype 171503364 +datatype[].id -372512406 +datatype[].maptype[].keytype 0 +datatype[].maptype[].valtype 1707615575 +datatype[].id 1416345047 +datatype[].arraytype[].datatype -372512406 datatype[].id 348447225 datatype[].structtype[].name "types.body" datatype[].structtype[].version 0 @@ -222,7 +222,7 @@ datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "complexarray" -datatype[].structtype[].field[].datatype 1100964733 +datatype[].structtype[].field[].datatype 1416345047 datatype[].structtype[].field[].detailedtype "" datatype[].id -853072901 datatype[].documenttype[].name "types" diff --git a/config-model/src/test/derived/types/types.sd b/config-model/src/test/derived/types/types.sd index 0590625e1c6..c908b648340 100644 --- a/config-model/src/test/derived/types/types.sd +++ b/config-model/src/test/derived/types/types.sd @@ -87,7 +87,7 @@ search types { field arrarr type array<array<array<string>>> {header} field maparr type array<map<string, string>> {header} - field complexarray type array< map<array<array<string>>, int> > {body} + field complexarray type array< map<int, array<array<string>>> > {body} struct mystruct { field bytearr type array<byte>{} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 4f99922a422..6a1e5b207c6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -16,6 +16,9 @@ import java.util.List; import static org.junit.Assert.assertEquals; +/** + * @author lesters + */ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase { @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java new file mode 100644 index 00000000000..d6e31ac8934 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +/** + * @author lesters + */ +public class DisallowComplexMapAndWsetKeyTypesTestCase { + + @Test(expected = IllegalArgumentException.class) + public void requireThatComplexTypesForMapKeysFail() throws ParseException { + testFieldType("map<mystruct,string>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatComplexTypesForWsetFail() throws ParseException { + testFieldType("weightedset<mystruct>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatNestedComplexTypesForMapFail() throws ParseException { + testFieldType("array<map<mystruct,string>>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatNestedComplexTypesForWsetFail() throws ParseException { + testFieldType("array<weightedset<mystruct>>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatDeepNestedComplexTypesForMapFail() throws ParseException { + testFieldType("map<string,map<mystruct,string>>"); + } + + private void testFieldType(String fieldType) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " struct mystruct {}\n" + + " field a type " + fieldType + " {}\n" + + " }\n" + + "}\n"); + builder.build(); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index d56a7c3e298..44bd7ec3708 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -9,43 +9,53 @@ package com.yahoo.config.provision; public enum NodeType { /** A node to be assigned to a tenant to run application workloads */ - tenant(false, "Tenant node"), + tenant(null, "Tenant node"), /** A host of a set of (Docker) tenant nodes */ - host(true, "Tenant docker host"), + host(tenant, "Tenant docker host"), /** Nodes running the shared proxy layer */ - proxy(false, "Proxy node"), + proxy(null, "Proxy node"), /** A host of a (Docker) proxy node */ - proxyhost(true, "Proxy docker host"), + proxyhost(proxy, "Proxy docker host"), /** A config server */ - config(false, "Config server"), + config(null, "Config server"), /** A host of a (Docker) config server node */ - confighost(true, "Config docker host"), + confighost(config, "Config docker host"), /** A controller */ - controller(false, "Controller"), + controller(null, "Controller"), /** A host of a (Docker) controller node */ - controllerhost(true, "Controller host"); + controllerhost(controller, "Controller host"); - private final boolean isDockerHost; + private final NodeType childNodeType; private final String description; - NodeType(boolean isDockerHost, String description) { - this.isDockerHost = isDockerHost; + NodeType(NodeType childNodeType, String description) { + this.childNodeType = childNodeType; this.description = description; } public boolean isDockerHost() { - return isDockerHost; + return childNodeType != null; } public String description() { return description; } + /** + * @return {@link NodeType} of the node(s) that run on this host + * @throws IllegalStateException if this type is not a host + */ + public NodeType childNodeType() { + if (! isDockerHost()) + throw new IllegalStateException(this + " has no children"); + + return childNodeType; + } } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 60dd7b0cea2..511cde23e89 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -192,5 +192,7 @@ <preprocess:include file='configserver-config.xml' required='false' /> <preprocess:include file='configserver-components.xml' required='false' /> + + <preprocess:include file='zookeeper-server-config.xml' required='false' /> </jdisc> </services> diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 7ad59d66916..76ea4579244 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -4,7 +4,7 @@ package com.yahoo.container.handler; import org.json.JSONException; import org.json.JSONObject; -import javax.xml.bind.DatatypeConverter; +import java.util.Base64; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -29,7 +29,7 @@ public class LogReader { for(File child : files) { long logTime = Files.readAttributes(child.toPath(), BasicFileAttributes.class).creationTime().toMillis(); if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) { - json.put(filename + child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); + json.put(filename + child.getName(), Base64.getEncoder().encodeToString(Files.readAllBytes(child.toPath()))); } else if (!child.isFile()){ traverse_folder(child, json, filename + child.getName() + "-"); diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index 45214d4d3df..d2950078868 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.logging.Logger; import java.util.stream.Collectors; import com.yahoo.search.config.IndexInfoConfig; @@ -20,6 +21,8 @@ import com.yahoo.container.QrSearchersConfig; */ public final class IndexModel { + private static final Logger log = Logger.getLogger(IndexModel.class.getName()); + // Copied from MasterClustersInfoUpdater. It's a temporary workaround for IndexFacts private Map<String, List<String>> masterClusters; private Map<String, SearchDefinition> searchDefinitions; @@ -112,8 +115,15 @@ public final class IndexModel { union.getOrCreateIndex(index.getName()); for (String command : index.allCommands()) union.addCommand(index.getName(), command); - for (String alias : index.aliases()) - union.addAlias(alias, index.getName()); + for (String alias : index.aliases()) { + try { + union.addAlias(alias, index.getName()); + } + catch (IllegalArgumentException e) { + log.fine("Conflicting alias '" + alias + " of " + index + " in " + sd + + " will not take effect for queries which does not specify that search definition"); + } + } } } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java index d38c481e33b..a679b17b6fc 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java @@ -76,10 +76,6 @@ public class HttpProviderSpec { public final Integer ycaRetryWait; public final Node ycaProxy; - //TODO:remove this - public final double cacheWeight; - - public static BundleInstantiationSpecification toBundleInstantiationSpecification(Type type) { return BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(type.className, null); } @@ -103,9 +99,6 @@ public class HttpProviderSpec { Integer cacheSizeMB, ConnectionParameters connectionParameters) { - final double defaultCacheWeight = 1.0d; - this.cacheWeight = (cacheWeight != null) ? cacheWeight : defaultCacheWeight; - this.path = path; this.nodes = unmodifiable(nodes); this.ycaApplicationId = ycaApplicationId; @@ -119,7 +112,7 @@ public class HttpProviderSpec { private List<HttpProviderSpec.Node> unmodifiable(List<HttpProviderSpec.Node> nodes) { return nodes == null ? - Collections.<HttpProviderSpec.Node>emptyList() : + Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(nodes)); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index 88c06635520..e3a5ce76ffb 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -244,30 +244,6 @@ public class IndexFactsTestCase { assertTrue(session.getIndex("e").isExact()); } - @Test - public void testOverlappingAliases() { - IndexInfoConfig cfg = new IndexInfoConfig(new IndexInfoConfig.Builder() - .indexinfo( - new Indexinfo.Builder() - .name("music2") - .command( - new Command.Builder().indexname( - "btitle").command("index")) - .alias(new Alias.Builder().alias("title") - .indexname("btitle"))).indexinfo( - new Indexinfo.Builder().name("music").command( - new Command.Builder().indexname("title") - .command("index")))); - try { - new IndexModel(cfg, (QrSearchersConfig) null); - fail("Excepted exception"); // (This is validated at deploy time) - } - catch (IllegalArgumentException e) { - assertEquals("Tried adding the alias 'title' for the index name 'btitle' when the name 'title' already maps to 'title'", - e.getMessage()); - } - } - private Query newQuery(String queryString, IndexFacts indexFacts) { Query query = new Query(queryString); query.getModel().setExecution(new Execution(new Execution.Context(null, indexFacts, null, null, null))); @@ -329,5 +305,21 @@ public class IndexFactsTestCase { assertEquals("url:\"https foo bar\"", query1.getModel().getQueryTree().toString()); assertEquals("url:\"https foo bar\"", query2.getModel().getQueryTree().toString()); } + + @Test + public void testConflictingAliases() { + SearchDefinition first = new SearchDefinition("first"); + Index field1 = new Index("field1"); + first.addIndex(field1); + + SearchDefinition second = new SearchDefinition("second"); + Index field2 = new Index("field2"); + field2.addAlias("field1"); + second.addIndex(field2); + + // Alias to field1 conflics with field1 in the "union" search definition. + // Should not produce an exception (but a log message): + new IndexFacts(new IndexModel(Collections.emptyMap(), ImmutableList.of(first, second))); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 7deb7712b28..15ee3724b6c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -27,11 +27,6 @@ public interface ConfigServer { PrepareResponse prepareResponse(); } - // TODO: Deprecated, remove when implementations have been removed - default PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content) { - return deploy(deployment, deployOptions, rotationCnames, rotationNames, content); - } - PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content); void restart(DeploymentId deployment, Optional<Hostname> hostname) throws NoInstanceException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 776cb86fbac..26110c9800e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -323,7 +323,7 @@ public class ApplicationController { ? triggered.sourceApplication().orElse(triggered.application()) : triggered.application(); - if (application.get().deploymentJobs().builtInternally()) { + if (application.get().deploymentJobs().deployedInternally()) { applicationPackage = new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion.id())); } else { applicationPackage = new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())); @@ -332,7 +332,7 @@ public class ApplicationController { } // Update application with information from application package - if ( ! preferOldestVersion && ! application.get().deploymentJobs().builtInternally()) + if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally()) application = storeWithUpdatedConfig(application, applicationPackage); // Assign global rotation diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 65ba7e68d31..23826a47931 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -114,7 +114,7 @@ public class DeploymentJobs { public Optional<IssueId> issueId() { return issueId; } - public boolean builtInternally() { return builtInternally; } + public boolean deployedInternally() { return builtInternally; } private static OptionalLong requireId(OptionalLong id, String message) { Objects.requireNonNull(id, message); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 846766c38a0..ef74498a50f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -108,9 +108,14 @@ public class DeploymentTrigger { triggering = JobRun.triggering(application.get().oldestDeployedPlatform().orElse(controller.systemVersion()), applicationVersion, Optional.empty(), Optional.empty(), "Application commit", clock.instant()); if (report.success()) { - if (acceptNewApplicationVersion(application.get())) + if (acceptNewApplicationVersion(application.get())) { application = application.withChange(application.get().change().with(applicationVersion)) .withOutstandingChange(Change.empty()); + if (application.get().deploymentJobs().deployedInternally()) + for (Run run : jobs.active()) + if (run.id().application().equals(report.applicationId())) + jobs.abort(run.id()); + } else application = application.withOutstandingChange(Change.of(applicationVersion)); } @@ -176,7 +181,7 @@ public class DeploymentTrigger { log.log(LogLevel.INFO, String.format("Triggering %s: %s", job, job.triggering)); try { applications().lockOrThrow(job.applicationId(), application -> { - if (application.get().deploymentJobs().builtInternally()) + if (application.get().deploymentJobs().deployedInternally()) jobs.start(job.applicationId(), job.jobType, new Versions(job.triggering.platform(), job.triggering.application(), job.triggering.sourcePlatform(), @@ -201,7 +206,7 @@ public class DeploymentTrigger { public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType, String user) { Application application = applications().require(applicationId); if (jobType == component) { - if (application.deploymentJobs().builtInternally()) + if (application.deploymentJobs().deployedInternally()) throw new IllegalArgumentException(applicationId + " has no component job we can trigger."); buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); @@ -411,7 +416,7 @@ public class DeploymentTrigger { } private JobState jobStateOf(Application application, JobType jobType) { - if (application.deploymentJobs().builtInternally()) { + if (application.deploymentJobs().deployedInternally()) { Optional<Run> run = controller.jobController().last(application.id(), jobType); return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1acf9bef363..e7ffa5115f1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -432,7 +432,7 @@ public class InternalStepRunner implements StepRunner { private DeploymentJobs.JobReport report(Run run) { return new DeploymentJobs.JobReport(run.id().application(), run.id().type(), - 1, + controller.applications().require(run.id().application()).deploymentJobs().projectId().orElse(1), run.id().number(), Optional.empty(), run.hasFailed() ? Optional.of(DeploymentJobs.JobError.unknown) : Optional.empty()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 453447830bb..1b45041abce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -133,7 +133,7 @@ public class JobController { /** Returns a list of all application which have registered. */ public List<ApplicationId> applications() { return copyOf(controller.applications().asList().stream() - .filter(application -> application.deploymentJobs().builtInternally()) + .filter(application -> application.deploymentJobs().deployedInternally()) .map(Application::id) .iterator()); } @@ -212,11 +212,11 @@ public class JobController { /** * Accepts and stores a new application package and test jar pair under a generated application version key. */ - public ApplicationVersion submit(ApplicationId id, SourceRevision revision, + public ApplicationVersion submit(ApplicationId id, SourceRevision revision, long projectId, byte[] packageBytes, byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { - if ( ! application.get().deploymentJobs().builtInternally()) { + if ( ! application.get().deploymentJobs().deployedInternally()) { // Copy all current packages to the new application store application.get().deployments().values().stream() .map(Deployment::applicationVersion) @@ -239,7 +239,7 @@ public class JobController { controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes)); - notifyOfNewSubmission(id, revision, run); + notifyOfNewSubmission(id, projectId, revision, run); }); return version.get(); } @@ -247,7 +247,7 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions) { controller.applications().lockIfPresent(id, application -> { - if ( ! application.get().deploymentJobs().builtInternally()) + if ( ! application.get().deploymentJobs().deployedInternally()) throw new IllegalArgumentException(id + " is not built here!"); locked(id, type, __ -> { @@ -330,10 +330,10 @@ public class JobController { } // TODO jvenstad: Find a more appropriate way of doing this when this is the only build service. - private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) { + private void notifyOfNewSubmission(ApplicationId id, long projectId, SourceRevision revision, long number) { DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id, JobType.component, - 1, + projectId, number, Optional.of(revision), Optional.empty()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 1452e3aa61d..7b615249a65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -235,7 +235,7 @@ public class ApplicationSerializer { deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField)); deploymentJobs.issueId().ifPresent(jiraIssueId -> cursor.setString(issueIdField, jiraIssueId.value())); - cursor.setBool(builtInternallyField, deploymentJobs.builtInternally()); + cursor.setBool(builtInternallyField, deploymentJobs.deployedInternally()); } private void jobStatusToSlime(Collection<JobStatus> jobStatuses, Cursor jobStatusArray) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 2058c556464..653398529c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -212,6 +212,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); @@ -396,6 +397,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .steps(application.deploymentSpec()) .sortedJobs(application.deploymentJobs().jobStatus().values()); + object.setBool("deployedInternally", application.deploymentJobs().deployedInternally()); Cursor deploymentsArray = object.setArray("deploymentJobs"); for (JobStatus job : jobStatus) { Cursor jobObject = deploymentsArray.addObject(); @@ -1246,6 +1248,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); SourceRevision sourceRevision = toSourceRevision(submitOptions).orElseThrow(() -> new IllegalArgumentException("Must specify 'repository', 'branch', and 'commit'")); + long projectId = Math.max(1, submitOptions.field("projectId").asLong()); ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); if ( ! applicationPackage.deploymentSpec().athenzDomain().isPresent()) @@ -1253,9 +1256,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { verifyApplicationIdentityConfiguration(tenant, Optional.of(applicationPackage)); return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, - sourceRevision, - applicationPackage.zippedContent(), - dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); + sourceRevision, + projectId, + applicationPackage.zippedContent(), + dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 05c7bd5e11b..2e67f4b2efc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.net.URI; @@ -394,9 +395,12 @@ class JobControllerApiHandlerHelper { * @return Response with the new application version */ static HttpResponse submitResponse(JobController jobController, String tenant, String application, - SourceRevision sourceRevision, byte[] appPackage, byte[] testPackage) { + SourceRevision sourceRevision, long projectId, byte[] appPackage, byte[] testPackage) { ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"), - sourceRevision, appPackage, testPackage); + sourceRevision, + projectId, + appPackage, + testPackage); Slime slime = new Slime(); Cursor responseObject = slime.setObject(); @@ -404,5 +408,19 @@ class JobControllerApiHandlerHelper { return new SlimeJsonResponse(slime); } + /** Aborts any job of the given type. */ + static HttpResponse abortJobResponse(JobController jobs, ApplicationId id, JobType type) { + Slime slime = new Slime(); + Cursor responseObject = slime.setObject(); + Optional<Run> run = jobs.last(id, type).flatMap(last -> jobs.active(last.id())); + if (run.isPresent()) { + jobs.abort(run.get().id()); + responseObject.setString("message", "Aborting " + run); + } + else + responseObject.setString("message", "Nothing to abort."); + return new SlimeJsonResponse(slime); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 33d53b0becf..378dc0315d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -128,20 +128,21 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private static boolean isTenantAdminOperation(Path path, Method method) { if (isHostedOperatorOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"); + path.matches("/application/v4/tenant/{tenant}/application/{application}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{job}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"); } private static boolean isTenantPipelineOperation(Path path, Method method) { if (isTenantAdminOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/submit") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/promote") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/prod/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/test/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/staging/{*}"); + path.matches("/application/v4/tenant/{tenant}/application/{application}/submit") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/promote") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/prod/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/test/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/staging/{*}"); } private void verifyIsHostedOperator(AthenzPrincipal principal) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index 332c440d18e..09f8de40378 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -65,7 +65,7 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple // Key store containing key pair from secret store factory.setKeyStore(KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(getClass().getSimpleName(), privateKey(), certificate()) + .withKeyEntry(getClass().getSimpleName(), privateKey(), certificates()) .build()); factory.setKeyStorePassword(""); @@ -77,8 +77,11 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple return KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(config.privateKeySecret())); } - /** Get certificate from secret store */ - private List<X509Certificate> certificate() { + /** + * Get certificate from secret store. If certificate secret contains multiple certificates, e.g. intermediate + * certificates, the entire chain will be read + */ + private List<X509Certificate> certificates() { return X509CertificateUtils.certificateListFromPem(secretStore.getSecret(config.certificateSecret())); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 991d26d2d1b..1bec06ea0e0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -153,7 +153,7 @@ public class ControllerTest { .region("deep-space-9") .build(); try { - tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, applicationPackage.zippedContent(), new byte[0]); + tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, 2, applicationPackage.zippedContent(), new byte[0]); fail("Expected exception due to illegal deployment spec."); } catch (IllegalArgumentException e) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 1eab8dece42..046d4af3899 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -31,7 +33,9 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.function.Supplier; @@ -113,6 +117,68 @@ public class DeploymentTriggerTest { } @Test + public void abortsInternalJobsOnNewApplicationChange() { + InternalDeploymentTester iTester = new InternalDeploymentTester(); + tester = iTester.tester(); + + Application app = iTester.app(); + ApplicationPackage applicationPackage = InternalDeploymentTester.applicationPackage; + + tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(app, true, systemTest); + tester.deployAndNotify(app, true, stagingTest); + tester.assertRunning(productionUsCentral1, app.id()); + + // Jobs run externally are not affected. + tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit(); + tester.assertRunning(productionUsCentral1, app.id()); + + tester.applications().deploymentTrigger().cancelChange(app.id(), false); + tester.deployAndNotify(app, false, systemTest); + tester.deployAndNotify(app, false, stagingTest); + tester.deployAndNotify(app, false, productionUsCentral1); + assertEquals(Change.empty(), tester.application(app.id()).change()); + tester.assertNotRunning(systemTest, app.id()); + tester.assertNotRunning(stagingTest, app.id()); + tester.assertNotRunning(productionUsCentral1, app.id()); + + RunId id = iTester.newRun(productionUsCentral1); + assertTrue(iTester.jobs().active(id).isPresent()); + + // Jobs run internally are aborted. + iTester.newSubmission(); + assertTrue(iTester.jobs().active(id).isPresent()); + iTester.runner().run(); + assertFalse(iTester.jobs().active(id).isPresent()); + + tester.readyJobTrigger().maintain(); + assertEquals(EnumSet.of(systemTest, stagingTest), iTester.jobs().active().stream() + .map(run -> run.id().type()) + .collect(Collectors.toCollection(() -> EnumSet.noneOf(JobType.class)))); + + iTester.runJob(JobType.systemTest); + iTester.runJob(JobType.stagingTest); + tester.readyJobTrigger().maintain(); + iTester.runJob(JobType.productionUsCentral1); + tester.readyJobTrigger().maintain(); + iTester.runJob(JobType.productionUsWest1); + iTester.runJob(JobType.productionUsEast3); + assertEquals(Change.empty(), iTester.app().change()); + + tester.upgradeSystem(new Version("8.9")); + iTester.runJob(JobType.systemTest); + iTester.runJob(JobType.stagingTest); + tester.readyJobTrigger().maintain(); + + // Jobs run internally are not aborted when the new submission is delayed. + iTester.newSubmission(); + iTester.runner().run(); + assertEquals(EnumSet.of(productionUsCentral1), iTester.jobs().active().stream() + .map(run -> run.id().type()) + .collect(Collectors.toCollection(() -> EnumSet.noneOf(JobType.class)))); + } + + @Test public void deploymentSpecDecidesTriggerOrder() { TenantName tenant = tester.controllerTester().createTenant("tenant1", "domain1", 1L); MockBuildService mockBuildService = tester.buildService(); @@ -798,6 +864,7 @@ public class DeploymentTriggerTest { assertTrue("All jobs consumed", tester.buildService().jobs().isEmpty()); assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures()); } + @Test public void testRetriesJobsFailingForCurrentChange() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index d5bcb205641..0549a2e4226 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -82,7 +82,7 @@ public class InternalDeploymentTester { * Submits a new application, and returns the version of the new submission. */ public ApplicationVersion newSubmission() { - ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, applicationPackage.zippedContent(), new byte[0]); + ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, 2, applicationPackage.zippedContent(), new byte[0]); tester.applicationStore().putApplicationPackage(appId, version.id(), applicationPackage.zippedContent()); tester.applicationStore().putTesterPackage(testerOf(appId), version.id(), new byte[0]); return version; @@ -109,7 +109,7 @@ public class InternalDeploymentTester { ApplicationVersion applicationVersion = newSubmission(); assertFalse(app().deployments().values().stream() - .anyMatch(deployment -> deployment.applicationVersion().equals(applicationVersion))); + .anyMatch(deployment -> deployment.applicationVersion().equals(applicationVersion))); assertEquals(applicationVersion, app().change().application().get()); assertFalse(app().change().platform().isPresent()); @@ -128,7 +128,7 @@ public class InternalDeploymentTester { public void deployNewPlatform(Version version) { tester.upgradeSystem(version); assertFalse(app().deployments().values().stream() - .anyMatch(deployment -> deployment.version().equals(version))); + .anyMatch(deployment -> deployment.version().equals(version))); assertEquals(version, app().change().platform().get()); assertFalse(app().change().application().isPresent()); @@ -230,7 +230,7 @@ public class InternalDeploymentTester { * Creates and submits a new application, and then starts the job of the given type. */ public RunId newRun(JobType type) { - assertFalse(app().deploymentJobs().builtInternally()); // Use this only once per test. + assertFalse(app().deploymentJobs().deployedInternally()); // Use this only once per test. newSubmission(); tester.readyJobTrigger().maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index e506c8c56ca..13ac5c41644 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -82,7 +82,7 @@ public class JobRunnerTest { phasedExecutor(phaser), stepRunner); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); jobs.start(id, systemTest, versions); try { @@ -113,7 +113,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); @@ -197,7 +197,7 @@ public class JobRunnerTest { Executors.newFixedThreadPool(32), waitingRunner(barrier)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); RunId runId = new RunId(id, systemTest, 1); jobs.start(id, systemTest, versions); @@ -233,7 +233,7 @@ public class JobRunnerTest { inThreadExecutor(), (id, step) -> Optional.of(running)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); for (int i = 0; i < jobs.historyLength(); i++) { jobs.start(id, systemTest, versions); @@ -261,7 +261,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 07cef02501d..ab00ecab376 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -424,6 +424,10 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationSubmissionData(packageWithService)), "{\"version\":\"1.0.43-d00d\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Nothing to abort.\"}"); + // PUT (create) the authenticated user byte[] data = new byte[0]; tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 7d379911238..293a1f8a746 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -48,6 +48,7 @@ public class JobControllerApiHandlerHelperTest { // Revision 1 gets deployed everywhere. ApplicationVersion revision1 = tester.deployNewSubmission(); + assertEquals(2, tester.app().deploymentJobs().projectId().getAsLong()); tester.clock().advance(Duration.ofMillis(1000)); // Revision 2 gets deployed everywhere except in us-east-3. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index 07a3dbb7f95..7bb78aba459 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -2,6 +2,7 @@ "application": "application1", "instance": "default", "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 0d7607f1df6..2f25e9471aa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -12,6 +12,7 @@ } } }, + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 4e4a870662a..70da148ef86 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -12,6 +12,7 @@ } } }, + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json index 837c46aaec1..56abc8f9d82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -15,6 +15,7 @@ } } }, + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 8e934001381..afe14cf1e6a 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -2,9 +2,11 @@ package com.yahoo.document.json; import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.Field; import com.yahoo.document.PositionDataType; +import com.yahoo.document.PrimitiveDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; @@ -186,7 +188,12 @@ public class JsonSerializationHelper { generator.writeStartObject(); for (Map.Entry<K, V> entry : map.entrySet()) { - generator.writeFieldName(entry.getKey().toString()); + K key = entry.getKey(); + DataType keyType = key.getDataType(); + if ( ! (keyType instanceof PrimitiveDataType)) { + throw new IllegalArgumentException("Can't use complex types as keys for map fields. Type: " + keyType); + } + generator.writeFieldName(key.toString()); entry.getValue().serialize(null, fieldWriter); } diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index b54f822c22e..90dff0982af 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -26,11 +26,9 @@ search book { } field mywsfloat type weightedset<float> { indexing: attribute - attribute: prefetch } field myarrayint type array<int> { indexing: attribute - attribute: prefetch } field mytriplearray type array<array<array<int>>> { diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java index 54f1ee3042f..54bc6239cf5 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java @@ -2,12 +2,9 @@ package com.yahoo.jdisc.http.filter.security.cors; import com.google.common.collect.ImmutableMap; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.jdisc.Response; import java.time.Duration; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -24,7 +21,7 @@ class CorsLogic { static final Map<String, String> ACCESS_CONTROL_HEADERS = ImmutableMap.of( "Access-Control-Max-Age", CORS_PREFLIGHT_REQUEST_CACHE_TTL, - "Access-Control-Allow-Headers", "Origin,Content-Type,Accept,Yahoo-Principal-Auth", + "Access-Control-Allow-Headers", "Origin,Content-Type,Accept,Yahoo-Principal-Auth,Okta-Identity-Token,Okta-Access-Token,Okta-Refresh-Token", "Access-Control-Allow-Methods", "OPTIONS,GET,PUT,DELETE,POST", "Access-Control-Allow-Credentials", "true" ); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 26e61e8917b..0f0b704bba7 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -58,13 +58,13 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, private static Logger log = Logger.getLogger(MessageBus.class.getName()); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final ProtocolRepository protocolRepository = new ProtocolRepository(); - private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<Map<String, RoutingTable>>(null); - private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<String, MessageHandler>(); + private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<>(null); + private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<>(); private final Network net; private final Messenger msn; private final Resender resender; - private int maxPendingCount = 0; - private int maxPendingSize = 0; + private int maxPendingCount; + private int maxPendingSize; private int pendingCount = 0; private int pendingSize = 0; private final Thread careTaker = new Thread(this::sendBlockedMessages); @@ -440,7 +440,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, @Override public void setupRouting(RoutingSpec spec) { - Map<String, RoutingTable> tables = new HashMap<String, RoutingTable>(); + Map<String, RoutingTable> tables = new HashMap<>(); for (int i = 0, len = spec.getNumTables(); i < len; ++i) { RoutingTableSpec table = spec.getTable(i); String name = table.getProtocol(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 6ef5e9adca7..b9ea69cb116 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -24,6 +24,7 @@ public class Messenger implements Runnable { private final AtomicBoolean destroyed = new AtomicBoolean(false); private final List<Task> children = new ArrayList<>(); private final Queue<Task> queue = new ArrayDeque<>(); + private final Thread thread = new Thread(this, "Messenger"); public Messenger() { @@ -38,7 +39,7 @@ public class Messenger implements Runnable { * * @param task The task to add. */ - public void addRecurrentTask(final Task task) { + void addRecurrentTask(final Task task) { children.add(task); } @@ -61,7 +62,11 @@ public class Messenger implements Runnable { * @param handler The handler to send to. */ public void deliverMessage(final Message msg, final MessageHandler handler) { - enqueue(new MessageTask(msg, handler)); + if (destroyed.get()) { + msg.discard(); + } else { + handler.handleMessage(msg); + } } /** @@ -194,38 +199,13 @@ public class Messenger implements Runnable { /** * <p>This method is called when being executed.</p> */ - public void run(); + void run(); /** * <p>This method is called for all tasks, even if {@link #run()} was * never called.</p> */ - public void destroy(); - } - - private static class MessageTask implements Task { - - final MessageHandler handler; - Message msg; - - MessageTask(final Message msg, final MessageHandler handler) { - this.msg = msg; - this.handler = handler; - } - - @Override - public void run() { - final Message msg = this.msg; - this.msg = null; - handler.handleMessage(msg); - } - - @Override - public void destroy() { - if (msg != null) { - msg.discard(); - } - } + void destroy(); } private static class ReplyTask implements Task { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java index 0dcc1dd67fd..3049639586d 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java @@ -183,7 +183,7 @@ public class RoutingNode implements ReplyHandler { } /** - * This method mergs this node as ready for merge. If it has a parent routing node, its pending member is + * This method markss this node as ready for merge. If it has a parent routing node, its pending member is * decremented. If this causes the parent's pending count to reach zero, its {@link #notifyMerge()} method is * invoked. A special flag is used to make sure that failed resending avoids notifying parents of previously * resolved branches of the tree. @@ -205,7 +205,7 @@ public class RoutingNode implements ReplyHandler { /** * This method merges the content of all its children, and invokes itself on the parent node. If not all children - * are ready for merg, this method does nothing. The rationale for this is that the last child to receive a reply + * are ready for merge, this method does nothing. The rationale for this is that the last child to receive a reply * will propagate the merge upwards. Once this method reaches the root node, the reply is either scheduled for * resending or passed to the owning reply handler. */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 7b484dfc481..ec911cc5600 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import java.net.URI; import java.util.ArrayList; @@ -24,12 +22,6 @@ public class ConfigServerInfo { private final Map<String, URI> configServerURIs; private final AthenzService configServerIdentity; - // TODO: Remove - public ConfigServerInfo(ConfigServerConfig config) { - this(config.loadBalancerHost(), config.hosts(), config.scheme(), config.port(), - (AthenzService) AthenzIdentities.from(config.configserverAthenzIdentity())); - } - public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames, String scheme, int port, AthenzService configServerAthenzIdentity) { this.configServerHostNames = configServerHostNames; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java deleted file mode 100644 index aaadd3cb24e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesImpl; - -import java.net.URI; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Various utilities for getting values from node-admin's environment. Immutable. - * - * @author Øyvind Bakksjø - * @author hmusum - */ -public class Environment { - private final ConfigServerInfo configServerInfo; - private final String environment; - private final String region; - private final String system; - private final String cloud; - private final String parentHostHostname; - private final IPAddresses ipAddresses; - private final PathResolver pathResolver; - private final List<String> logstashNodes; - private final NodeType nodeType; - private final ContainerEnvironmentResolver containerEnvironmentResolver; - private final String certificateDnsSuffix; - private final URI ztsUri; - private final AthenzService nodeAthenzIdentity; - private final boolean nodeAgentCertEnabled; - private final Path trustStorePath; - private final DockerNetworking dockerNetworking; - - private Environment(ConfigServerInfo configServerInfo, - Path trustStorePath, - String environment, - String region, - String system, - String cloud, - String parentHostHostname, - IPAddresses ipAddresses, - PathResolver pathResolver, - List<String> logstashNodes, - NodeType nodeType, - ContainerEnvironmentResolver containerEnvironmentResolver, - String certificateDnsSuffix, - URI ztsUri, - AthenzService nodeAthenzIdentity, - boolean nodeAgentCertEnabled, - DockerNetworking dockerNetworking) { - this.configServerInfo = Objects.requireNonNull(configServerInfo, "configServerConfig cannot be null"); - this.environment = Objects.requireNonNull(environment, "environment cannot be null");; - this.region = Objects.requireNonNull(region, "region cannot be null");; - this.system = Objects.requireNonNull(system, "system cannot be null");; - this.cloud = Objects.requireNonNull(cloud, "cloud cannot be null"); - this.parentHostHostname = parentHostHostname; - this.ipAddresses = ipAddresses; - this.pathResolver = pathResolver; - this.logstashNodes = logstashNodes; - this.nodeType = nodeType; - this.containerEnvironmentResolver = containerEnvironmentResolver; - this.certificateDnsSuffix = certificateDnsSuffix; - this.ztsUri = ztsUri; - this.nodeAthenzIdentity = nodeAthenzIdentity; - this.nodeAgentCertEnabled = nodeAgentCertEnabled; - this.trustStorePath = trustStorePath; - this.dockerNetworking = Objects.requireNonNull(dockerNetworking, "dockerNetworking cannot be null"); - } - - public List<String> getConfigServerHostNames() { return configServerInfo.getConfigServerHostNames(); } - - public String getEnvironment() { return environment; } - - public String getRegion() { - return region; - } - - public String getSystem() { - return system; - } - - public String getCloud() { return cloud; } - - public String getParentHostHostname() { - return parentHostHostname; - } - - public String getZone() { - return getEnvironment() + "." + getRegion(); - } - - public IPAddresses getIpAddresses() { - return ipAddresses; - } - - public PathResolver getPathResolver() { - return pathResolver; - } - - /** - * Translates an absolute path in node agent container to an absolute path in node admin container. - * @param containerName name of the node agent container - * @param pathInNode absolute path in that container - * @return the absolute path in node admin container pointing at the same inode - */ - public Path pathInNodeAdminFromPathInNode(ContainerName containerName, Path pathInNode) { - if (! pathInNode.isAbsolute()) { - throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode); - } - - return pathResolver.getApplicationStoragePathForNodeAdmin() - .resolve(containerName.asString()) - .resolve(PathResolver.ROOT.relativize(pathInNode)); - } - - /** - * Translates an absolute path in node agent container to an absolute path in host. - * @param containerName name of the node agent container - * @param pathInNode absolute path in that container - * @return the absolute path in host pointing at the same inode - */ - public Path pathInHostFromPathInNode(ContainerName containerName, Path pathInNode) { - if (! pathInNode.isAbsolute()) { - throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode); - } - - return pathResolver.getApplicationStoragePathForHost() - .resolve(containerName.asString()) - .resolve(PathResolver.ROOT.relativize(pathInNode)); - } - - public Path pathInNodeUnderVespaHome(String relativePath) { - return pathResolver.getVespaHomePathForContainer() - .resolve(relativePath); - } - - public List<String> getLogstashNodes() { - return logstashNodes; - } - - public NodeType getNodeType() { return nodeType; } - - public ContainerEnvironmentResolver getContainerEnvironmentResolver() { - return containerEnvironmentResolver; - } - - public Path getTrustStorePath() { - return trustStorePath; - } - - public AthenzService getConfigserverAthenzIdentity() { - return configServerInfo.getConfigServerIdentity(); - } - - public AthenzService getNodeAthenzIdentity() { - return nodeAthenzIdentity; - } - - public String getCertificateDnsSuffix() { - return certificateDnsSuffix; - } - - public URI getZtsUri() { - return ztsUri; - } - - public URI getConfigserverLoadBalancerEndpoint() { - return configServerInfo.getLoadBalancerEndpoint(); - } - - public boolean isNodeAgentCertEnabled() { - return nodeAgentCertEnabled; - } - - public DockerNetworking getDockerNetworking() { - return dockerNetworking; - } - - public static class Builder { - private ConfigServerInfo configServerInfo; - private String environment; - private String region; - private String system; - private String cloud; - private String parentHostHostname; - private IPAddresses ipAddresses; - private PathResolver pathResolver; - private List<String> logstashNodes = Collections.emptyList(); - private NodeType nodeType = NodeType.tenant; - private ContainerEnvironmentResolver containerEnvironmentResolver; - private String certificateDnsSuffix; - private URI ztsUri; - private AthenzService nodeAthenzIdentity; - private boolean nodeAgentCertEnabled; - private Path trustStorePath; - private DockerNetworking dockerNetworking; - - public Builder configServerConfig(ConfigServerConfig configServerConfig) { - this.configServerInfo = new ConfigServerInfo(configServerConfig); - return this; - } - - public Builder configServerInfo(ConfigServerInfo configServerInfo) { - this.configServerInfo = configServerInfo; - return this; - } - - public Builder environment(String environment) { - this.environment = environment; - return this; - } - - public Builder region(String region) { - this.region = region; - return this; - } - - public Builder system(String system) { - this.system = system; - return this; - } - - public Builder cloud(String cloud) { - this.cloud = cloud; - return this; - } - - public Builder parentHostHostname(String parentHostHostname) { - this.parentHostHostname = parentHostHostname; - return this; - } - - public Builder ipAddresses(IPAddresses ipAddresses) { - this.ipAddresses = ipAddresses; - return this; - } - - public Builder pathResolver(PathResolver pathResolver) { - this.pathResolver = pathResolver; - return this; - } - - public Builder containerEnvironmentResolver(ContainerEnvironmentResolver containerEnvironmentResolver) { - this.containerEnvironmentResolver = containerEnvironmentResolver; - return this; - } - - public Builder logstashNodes(List<String> hosts) { - this.logstashNodes = hosts; - return this; - } - - public Builder nodeType(NodeType nodeType) { - this.nodeType = nodeType; - return this; - } - - public Builder certificateDnsSuffix(String certificateDnsSuffix) { - this.certificateDnsSuffix = certificateDnsSuffix; - return this; - } - - public Builder ztsUri(URI ztsUri) { - this.ztsUri = ztsUri; - return this; - } - - public Builder nodeAthenzIdentity(AthenzService nodeAthenzIdentity) { - this.nodeAthenzIdentity = nodeAthenzIdentity; - return this; - } - - public Builder enableNodeAgentCert(boolean nodeAgentCertEnabled) { - this.nodeAgentCertEnabled = nodeAgentCertEnabled; - return this; - } - - public Builder trustStorePath(Path trustStorePath) { - this.trustStorePath = trustStorePath; - return this; - } - - public Builder dockerNetworking(DockerNetworking dockerNetworking) { - this.dockerNetworking = dockerNetworking; - return this; - } - - public Environment build() { - return new Environment(configServerInfo, - trustStorePath, - environment, - region, - system, - cloud, - parentHostHostname, - Optional.ofNullable(ipAddresses).orElseGet(IPAddressesImpl::new), - Optional.ofNullable(pathResolver).orElseGet(PathResolver::new), - logstashNodes, - nodeType, - Optional.ofNullable(containerEnvironmentResolver).orElseGet(() -> node -> ""), - certificateDnsSuffix, - ztsUri, - nodeAthenzIdentity, - nodeAgentCertEnabled, - dockerNetworking); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java deleted file mode 100644 index 433fae0e551..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.vespa.defaults.Defaults; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * @author freva - */ -public class PathResolver { - public static final Path ROOT = Paths.get("/"); - public static final Path DEFAULT_HOST_ROOT = Paths.get("/host"); - public static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - - private final Path hostRoot; - private final Path vespaHomePathForContainer; - - private final Path applicationStoragePathForNodeAdmin; - private final Path applicationStoragePathForHost; - - /** - * @param hostRoot the absolute path to the root of the host's file system - * @param vespaHomeForContainer the absolute path of Vespa home in the mount namespace of any - * and all Docker containers managed by Node Admin. - */ - public PathResolver(Path hostRoot, Path vespaHomeForContainer) { - if (!hostRoot.isAbsolute()) { - throw new IllegalArgumentException("Path to root of host file system is not absolute: " + - hostRoot); - } - this.hostRoot = hostRoot; - - if (!vespaHomeForContainer.isAbsolute()) { - throw new IllegalArgumentException("Path to Vespa home is not absolute: " + vespaHomeForContainer); - } - this.vespaHomePathForContainer = vespaHomeForContainer; - - this.applicationStoragePathForNodeAdmin = hostRoot.resolve(RELATIVE_APPLICATION_STORAGE_PATH); - this.applicationStoragePathForHost = ROOT.resolve(RELATIVE_APPLICATION_STORAGE_PATH); - } - - public PathResolver() { - this(DEFAULT_HOST_ROOT, Paths.get(Defaults.getDefaults().vespaHome())); - } - - /** For testing */ - public PathResolver(Path vespaHomePathForContainer, Path applicationStoragePathForNodeAdmin, Path applicationStoragePathForHost) { - this.hostRoot = DEFAULT_HOST_ROOT; - this.vespaHomePathForContainer = vespaHomePathForContainer; - this.applicationStoragePathForNodeAdmin = applicationStoragePathForNodeAdmin; - this.applicationStoragePathForHost = applicationStoragePathForHost; - } - - /** - * Returns the absolute path of the Vespa home directory in any Docker container mount namespace. - * - * It's a limitation of current implementation that all containers MUST have the same Vespa - * home directory path. - */ - public Path getVespaHomePathForContainer() { - return vespaHomePathForContainer; - } - - /** Returns the absolute path to the container storage directory for the node admin (this process). */ - public Path getApplicationStoragePathForNodeAdmin() { - return applicationStoragePathForNodeAdmin; - } - - /** Returns the absolute path to the container storage directory for the host. */ - public Path getApplicationStoragePathForHost() { - return applicationStoragePathForHost; - } - - /** - * Returns the absolute path to the directory which is the root directory of the host - * file system. - */ - public Path getPathToRootOfHost() { - return hostRoot; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java deleted file mode 100644 index 15cfa99c749..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.config; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 7a83f00e297..41fe67f1996 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -304,9 +304,9 @@ public class DockerOperationsImpl implements DockerOperations { context.pathInNodeUnderVespaHome("tmp"), context.pathInNodeUnderVespaHome("var/container-data"))); - if (context.nodeType() == NodeType.proxyhost) + if (context.nodeType() == NodeType.proxy) paths.add(context.pathInNodeUnderVespaHome("var/vespa-hosted/routing")); - if (context.nodeType() == NodeType.host) + if (context.nodeType() == NodeType.tenant) paths.add(varLibSia); paths.forEach(path -> command.withVolume(context.pathOnHostFromPathInNode(path), path)); @@ -316,18 +316,18 @@ public class DockerOperationsImpl implements DockerOperations { if (isInfrastructureHost(context.nodeType())) command.withSharedVolume(varLibSia, varLibSia); - if (context.nodeType() == NodeType.proxyhost || context.nodeType() == NodeType.controllerhost) + if (context.nodeType() == NodeType.proxy || context.nodeType() == NodeType.controller) command.withSharedVolume(Paths.get("/opt/yahoo/share/ssl/certs"), Paths.get("/opt/yahoo/share/ssl/certs")); - if (context.nodeType() == NodeType.host) + if (context.nodeType() == NodeType.tenant) command.withSharedVolume(Paths.get("/var/zpe"), context.pathInNodeUnderVespaHome("var/zpe")); } /** Returns whether given nodeType is a Docker host for infrastructure nodes */ private static boolean isInfrastructureHost(NodeType nodeType) { - return nodeType == NodeType.confighost || - nodeType == NodeType.proxyhost || - nodeType == NodeType.controllerhost; + return nodeType == NodeType.config || + nodeType == NodeType.proxy || + nodeType == NodeType.controller; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java deleted file mode 100644 index ce751548f75..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.logging; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.component.Environment; - -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * @author mortent - */ -public class FilebeatConfigProvider { - - private static final String TENANT_FIELD = "%%TENANT%%"; - private static final String APPLICATION_FIELD = "%%APPLICATION%%"; - private static final String INSTANCE_FIELD = "%%INSTANCE%%"; - private static final String ENVIRONMENT_FIELD = "%%ENVIRONMENT%%"; - private static final String REGION_FIELD = "%%REGION%%"; - private static final String FILEBEAT_SPOOL_SIZE_FIELD = "%%FILEBEAT_SPOOL_SIZE%%"; - private static final String LOGSTASH_HOSTS_FIELD = "%%LOGSTASH_HOSTS%%"; - private static final String LOGSTASH_WORKERS_FIELD = "%%LOGSTASH_WORKERS%%"; - private static final String LOGSTASH_BULK_MAX_SIZE_FIELD = "%%LOGSTASH_BULK_MAX_SIZE%%"; - - private static final int logstashWorkers = 3; - private static final int logstashBulkMaxSize = 2048; - private final Environment environment; - - public FilebeatConfigProvider(Environment environment) { - this.environment = environment; - } - - public Optional<String> getConfig(NodeAgentContext context, NodeSpec node) { - - if (environment.getLogstashNodes().size() == 0 || !node.getOwner().isPresent()) { - return Optional.empty(); - } - NodeSpec.Owner owner = node.getOwner().get(); - int spoolSize = environment.getLogstashNodes().size() * logstashWorkers * logstashBulkMaxSize; - String logstashNodeString = environment.getLogstashNodes().stream() - .map(this::addQuotes) - .collect(Collectors.joining(",")); - return Optional.of(getTemplate(context) - .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment()) - .replaceAll(REGION_FIELD, environment.getRegion()) - .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize)) - .replaceAll(LOGSTASH_HOSTS_FIELD, logstashNodeString) - .replaceAll(LOGSTASH_WORKERS_FIELD, Integer.toString(logstashWorkers)) - .replaceAll(LOGSTASH_BULK_MAX_SIZE_FIELD, Integer.toString(logstashBulkMaxSize)) - .replaceAll(TENANT_FIELD, owner.getTenant()) - .replaceAll(APPLICATION_FIELD, owner.getApplication()) - .replaceAll(INSTANCE_FIELD, owner.getInstance())); - } - - private String addQuotes(String logstashNode) { - return logstashNode.startsWith("\"") - ? logstashNode - : String.format("\"%s\"", logstashNode); - } - - private String getTemplate(NodeAgentContext context) { - return "################### Filebeat Configuration Example #########################\n" + - "\n" + - "############################# Filebeat ######################################\n" + - "filebeat:\n" + - " # List of prospectors to fetch data.\n" + - " prospectors:\n" + - "\n" + - " # vespa\n" + - " - paths:\n" + - " - " + context.pathInNodeUnderVespaHome("logs/vespa/vespa.log") + "\n" + - " exclude_files: [\".gz$\"]\n" + - " document_type: vespa\n" + - " fields:\n" + - " HV-tenant: %%TENANT%%\n" + - " HV-application: %%APPLICATION%%\n" + - " HV-instance: %%INSTANCE%%\n" + - " HV-region: %%REGION%%\n" + - " HV-environment: %%ENVIRONMENT%%\n" + - " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + - " fields_under_root: true\n" + - " close_older: 20m\n" + - " force_close_files: true\n" + - "\n" + - " # vespa qrs\n" + - " - paths:\n" + - " - " + context.pathInNodeUnderVespaHome("logs/vespa/qrs/QueryAccessLog.*.*") + "\n" + - " exclude_files: [\".gz$\"]\n" + - " exclude_lines: [\"reserved-for-internal-use/feedapi\"]\n" + - " document_type: vespa-qrs\n" + - " fields:\n" + - " HV-tenant: %%TENANT%%\n" + - " HV-application: %%APPLICATION%%\n" + - " HV-instance: %%INSTANCE%%\n" + - " HV-region: %%REGION%%\n" + - " HV-environment: %%ENVIRONMENT%%\n" + - " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + - " fields_under_root: true\n" + - " close_older: 20m\n" + - " force_close_files: true\n" + - "\n" + - " # General filebeat configuration options\n" + - " #\n" + - " # Event count spool threshold - forces network flush if exceeded\n" + - " spool_size: %%FILEBEAT_SPOOL_SIZE%%\n" + - "\n" + - " # Defines how often the spooler is flushed. After idle_timeout the spooler is\n" + - " # Flush even though spool_size is not reached.\n" + - " #idle_timeout: 5s\n" + - " publish_async: false\n" + - "\n" + - " # Name of the registry file. Per default it is put in the current working\n" + - " # directory. In case the working directory is changed after when running\n" + - " # filebeat again, indexing starts from the beginning again.\n" + - " registry_file: /var/lib/filebeat/registry\n" + - "\n" + - " # Full Path to directory with additional prospector configuration files. Each file must end with .yml\n" + - " # These config files must have the full filebeat config part inside, but only\n" + - " # the prospector part is processed. All global options like spool_size are ignored.\n" + - " # The config_dir MUST point to a different directory then where the main filebeat config file is in.\n" + - " #config_dir:\n" + - "\n" + - "###############################################################################\n" + - "############################# Libbeat Config ##################################\n" + - "# Base config file used by all other beats for using libbeat features\n" + - "\n" + - "############################# Output ##########################################\n" + - "\n" + - "# Configure what outputs to use when sending the data collected by the beat.\n" + - "# Multiple outputs may be used.\n" + - "output:\n" + - "\n" + - " ### Logstash as output\n" + - " logstash:\n" + - " # The Logstash hosts\n" + - " hosts: [%%LOGSTASH_HOSTS%%]\n" + - "\n" + - " timeout: 15\n" + - "\n" + - " # Number of workers per Logstash host.\n" + - " worker: %%LOGSTASH_WORKERS%%\n" + - "\n" + - " # Set gzip compression level.\n" + - " compression_level: 3\n" + - "\n" + - " # Optional load balance the events between the Logstash hosts\n" + - " loadbalance: true\n" + - "\n" + - " # Optional index name. The default index name depends on the each beat.\n" + - " # For Packetbeat, the default is set to packetbeat, for Topbeat\n" + - " # top topbeat and for Filebeat to filebeat.\n" + - " #index: filebeat\n" + - "\n" + - " bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%%\n" + - "\n" + - " # Optional TLS. By default is off.\n" + - " #tls:\n" + - " # List of root certificates for HTTPS server verifications\n" + - " #certificate_authorities: [\"/etc/pki/root/ca.pem\"]\n" + - "\n" + - " # Certificate for TLS client authentication\n" + - " #certificate: \"/etc/pki/client/cert.pem\"\n" + - "\n" + - " # Client Certificate Key\n" + - " #certificate_key: \"/etc/pki/client/cert.key\"\n" + - "\n" + - " # Controls whether the client verifies server certificates and host name.\n" + - " # If insecure is set to true, all server host names and certificates will be\n" + - " # accepted. In this mode TLS based connections are susceptible to\n" + - " # man-in-the-middle attacks. Use only for testing.\n" + - " #insecure: true\n" + - "\n" + - " # Configure cipher suites to be used for TLS connections\n" + - " #cipher_suites: []\n" + - "\n" + - " # Configure curve types for ECDHE based cipher suites\n" + - " #curve_types: []\n" + - "\n" + - "############################# Shipper #########################################\n" + - "\n" + - "shipper:\n" + - "\n" + - "############################# Logging #########################################\n" + - "\n" + - "# There are three options for the log ouput: syslog, file, stderr.\n" + - "# Under Windos systems, the log files are per default sent to the file output,\n" + - "# under all other system per default to syslog.\n" + - "logging:\n" + - "\n" + - " # Send all logging output to syslog. On Windows default is false, otherwise\n" + - " # default is true.\n" + - " to_syslog: false\n" + - "\n" + - " # Write all logging output to files. Beats automatically rotate files if rotateeverybytes\n" + - " # limit is reached.\n" + - " to_files: true\n" + - "\n" + - " # To enable logging to files, to_files option has to be set to true\n" + - " files:\n" + - " # The directory where the log files will written to.\n" + - " path: " + context.pathInNodeUnderVespaHome("logs/filebeat") + "\n" + - "\n" + - " # The name of the files where the logs are written to.\n" + - " name: filebeat\n" + - "\n" + - " # Configure log file size limit. If limit is reached, log file will be\n" + - " # automatically rotated\n" + - " rotateeverybytes: 10485760 # = 10MB\n" + - "\n" + - " # Number of rotated log files to keep. Oldest files will be deleted first.\n" + - " keepfiles: 7\n" + - "\n" + - " # Enable debug output for selected components. To enable all selectors use [\"*\"]\n" + - " # Other available selectors are beat, publish, service\n" + - " # Multiple selectors can be chained.\n" + - " #selectors: [ ]\n" + - "\n" + - " # Sets log level. The default log level is error.\n" + - " # Available log levels are: critical, error, warning, info, debug\n" + - " level: warning\n"; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java deleted file mode 100644 index cf010121c2a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class FileHelper { - private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName()); - - /** - * (Recursively) deletes files if they match all the criteria, also deletes empty directories. - * - * @param basePath Base path from where to start the search - * @param maxAge Delete files older (last modified date) than maxAge - * @param fileNameRegex Delete files where filename matches fileNameRegex - * @param recursive Delete files in sub-directories (with the same criteria) - */ - public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException { - Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - if (recursive) { - deleteFiles(path, maxAge, fileNameRegex, true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } else if (isPatternMatchingFilename(fileNamePattern, path) && - isTimeSinceLastModifiedMoreThan(path, maxAge)) { - if (! Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes all files in target directory except the n most recent (by modified date) - * - * @param basePath Base path to delete from - * @param nMostRecentToKeep Number of most recent files to keep - */ - static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException { - if (nMostRecentToKeep < 1) { - throw new IllegalArgumentException("Number of files to keep must be a positive number"); - } - - List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream() - .filter(Files::isRegularFile) - .sorted(Comparator.comparing(FileHelper::getLastModifiedTime)) - .skip(nMostRecentToKeep) - .collect(Collectors.toList()); - - for (Path path : pathsInDeleteDir) { - if (!Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - - static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException { - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - deleteFilesLargerThan(path, sizeInBytes); - } else { - if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes directories and their contents if they match all the criteria - * - * @param basePath Base path to delete the directories from - * @param maxAge Delete directories older (last modified date) than maxAge - * @param dirNameRegex Delete directories where directory name matches dirNameRegex - */ - public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException { - Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) { - boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path) - .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge)) - .orElse(true); - - if (mostRecentFileModifiedBeforeMaxAge) { - deleteFiles(path, Duration.ZERO, Optional.empty(), true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } - } - } - - /** - * Similar to rm -rf file: - * - It's not an error if file doesn't exist - * - If file is a directory, it and all content is removed - * - For symlinks: Only the symlink is removed, not what the symlink points to - */ - public static void recursiveDelete(Path basePath) throws IOException { - if (Files.isDirectory(basePath)) { - for (Path path : listContentsOfDirectory(basePath)) { - recursiveDelete(path); - } - } - - Files.deleteIfExists(basePath); - } - - public static void moveIfExists(Path from, Path to) throws IOException { - if (Files.exists(from)) { - Files.move(from, to); - } - } - - private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException { - return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime)); - } - - private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) { - Instant nowMinusDuration = Instant.now().minus(duration); - Instant lastModified = getLastModifiedTime(path).toInstant(); - - // Return true also if they are equal for test stability - // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration) - return !lastModified.isAfter(nowMinusDuration); - } - - private static boolean isPatternMatchingFilename(Pattern pattern, Path path) { - return pattern == null || pattern.matcher(path.getFileName().toString()).find(); - } - - /** - * @return list all files in a directory, returns empty list if directory does not exist - */ - public static List<Path> listContentsOfDirectory(Path basePath) { - try (Stream<Path> directoryStream = Files.list(basePath)) { - return directoryStream.collect(Collectors.toList()); - } catch (NoSuchFileException ignored) { - return Collections.emptyList(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); - } - } - - static FileTime getLastModifiedTime(Path path) { - try { - return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS); - } catch (IOException e) { - throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 6a9f0a69249..c9e4a8fac7d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -111,8 +111,8 @@ public class StorageMaintainer { //routing-configage Path routingAgeCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60, - routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(), - "-m", "90", "-a", "routing-configage"); + routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf.tmp").toString(), + "-m", "1", "-a", "routing-configage", "--ignore_file_not_found"); configs.add(annotatedCheck(context, node, routingAgeSchedule)); //ssl-check diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index bbea505b19d..22093258930 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -70,7 +70,7 @@ public class AthenzCredentialsMaintainer { private final CsrGenerator csrGenerator; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts - private Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); + private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); public AthenzCredentialsMaintainer(URI ztsEndpoint, Path trustStorePath, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 395b4d458a2..ef137c55ffb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -337,7 +337,7 @@ public class NodeAgentImpl implements NodeAgent { } if (node.getWantedDockerImage().isPresent() && !node.getWantedDockerImage().get().equals(existingContainer.image)) { return Optional.of("The node is supposed to run a new Docker image: " - + existingContainer + " -> " + node.getWantedDockerImage().get()); + + existingContainer.image.asString() + " -> " + node.getWantedDockerImage().get().asString()); } if (!existingContainer.state.isRunning()) { return Optional.of("Container no longer running"); diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def deleted file mode 100644 index 6a088829bad..00000000000 --- a/node-admin/src/main/resources/configdefinitions/config-server.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.hosted.node.admin.config - -hosts[] string -port int default=8080 range=[1,65535] -scheme string default="http" -loadBalancerHost string default="" -configserverAthenzIdentity string default="vespa.configserver"
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java deleted file mode 100644 index 281c7df9a1f..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.component; - -import org.junit.Test; - -import java.nio.file.Paths; - -import static org.junit.Assert.assertEquals; - -public class PathResolverTest { - @Test - public void testNodeAdminOnHost() { - PathResolver pathResolver = new PathResolver(Paths.get("/"), Paths.get("/home/y")); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost()); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin()); - assertEquals(Paths.get("/"), pathResolver.getPathToRootOfHost()); - assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer()); - } - - @Test - public void testNodeAdminInContainer() { - PathResolver pathResolver = new PathResolver(Paths.get("/host"), Paths.get("/home/y")); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost()); - assertEquals(Paths.get("/host/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin()); - assertEquals(Paths.get("/host"), pathResolver.getPathToRootOfHost()); - assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer()); - } -}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java deleted file mode 100644 index d4fadabe695..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.logging; - -import com.google.common.collect.ImmutableList; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.component.Environment; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.provision.Node; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.*; - -/** - * @author mortent - */ -public class FilebeatConfigProviderTest { - - private static final String tenant = "vespa"; - private static final String application = "music"; - private static final String instance = "default"; - private static final String environment = "prod"; - private static final String region = "us-north-1"; - private static final String system = "main"; - private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2"); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("node-123.hostname.tld").build(); - - @Test - public void it_replaces_all_fields_correctly() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - - assertTrue(config.isPresent()); - String configString = config.get(); - assertThat(configString, not(containsString("%%"))); - } - - @Test - public void it_does_not_generate_config_when_no_logstash_nodes() { - Environment env = getEnvironment(Collections.emptyList()); - - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(env); - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - assertFalse(config.isPresent()); - } - - @Test - public void it_does_not_generate_config_for_nodes_wihout_owner() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - NodeSpec node = new NodeSpec.Builder() - .flavor("flavor") - .state(Node.State.active) - .nodeType(NodeType.tenant) - .hostname("hostname") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - Optional<String> config = filebeatConfigProvider.getConfig(context, node); - assertFalse(config.isPresent()); - } - - @Test - public void it_generates_correct_index_source() { - assertThat(getConfigString(), containsString("index_source: \"hosted-instance_vespa_music_us-north-1_prod_default\"")); - } - - @Test - public void it_sets_logstash_nodes_properly() { - assertThat(getConfigString(), containsString("hosts: [\"logstash1\",\"logstash2\"]")); - } - - @Test - public void it_does_not_add_double_quotes() { - Environment environment = getEnvironment(ImmutableList.of("unquoted", "\"quoted\"")); - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - assertThat(config.get(), containsString("hosts: [\"unquoted\",\"quoted\"]")); - } - - @Test - public void it_generates_correct_spool_size() { - // 2 nodes, 3 workers, 2048 buffer size -> 12288 - assertThat(getConfigString(), containsString("spool_size: 12288")); - } - - private String getConfigString() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - NodeSpec node = createNodeRepositoryNode(tenant, application, instance); - return filebeatConfigProvider.getConfig(context, node).orElseThrow(() -> new RuntimeException("Failed to get filebeat config")); - } - - private Environment getEnvironment(List<String> logstashNodes) { - return new Environment.Builder() - .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) - .environment(environment) - .region(region) - .system(system) - .logstashNodes(logstashNodes) - .cloud("mycloud") - .dockerNetworking(DockerNetworking.HOST_NETWORK) - .build(); - } - - private NodeSpec createNodeRepositoryNode(String tenant, String application, String instance) { - NodeSpec.Owner owner = new NodeSpec.Owner(tenant, application, instance); - return new NodeSpec.Builder() - .owner(owner) - .flavor("flavor") - .state(Node.State.active) - .nodeType(NodeType.tenant) - .hostname("hostname") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java deleted file mode 100644 index 6b53bc217c4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author freva - */ -public class FileHelperTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void initFiles() throws IOException { - for (int i=0; i<10; i++) { - File temp = folder.newFile("test_" + i + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis()); - } - - for (int i=0; i<7; i++) { - File temp = folder.newFile("test_" + i + "_file.test"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis()); - } - - for (int i=0; i<5; i++) { - File temp = folder.newFile(i + "-abc" + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis()); - } - - File temp = folder.newFile("week_old_file.json"); - temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis()); - } - - @Test - public void testDeleteAll() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false); - - assertEquals(0, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file - } - - @Test - public void testDeleteSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false); - - assertEquals(7, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefixAndSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file - } - - @Test - public void testDeleteOld() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file - } - - @Test - public void testDeleteWithAllParameters() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false); - - assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json) - } - - @Test - public void testDeleteWithSubDirectoriesNoRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - // 6 test_*.json from test_folder1/ - // + 9 test_*.json and 4 abc_*.json from test_folder2/ - // + 13 test_*.json from test_folder2/subSubFolder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves - assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteWithSubDirectoriesRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true); - - // 4 abc_*.json from test_folder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder2/ itself - assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories - } - - @Test - public void testGetContentsOfNonExistingDirectory() { - Path fakePath = Paths.get("/some/made/up/dir/"); - assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath)); - } - - @Test(expected=IllegalArgumentException.class) - public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5); - } - - @Test - public void testDeleteFilesExceptFiveMostRecent() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5); - - assertEquals(5, getContentsOfDirectory(folder.getRoot()).length); - - String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"}; - String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot())) - .map(File::getName) - .sorted() - .toArray(String[]::new); - - assertArrayEquals(oldestFiles, remainingFiles); - } - - @Test - public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException { - String[] filesPreDelete = folder.getRoot().list(); - - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50); - - assertArrayEquals(filesPreDelete, folder.getRoot().list()); - } - - @Test - public void testDeleteFilesLargerThan10B() throws IOException { - initSubDirectories(); - - File temp1 = new File(folder.getRoot(), "small_file"); - writeNBytesToFile(temp1, 50); - - File temp2 = new File(folder.getRoot(), "some_file"); - writeNBytesToFile(temp2, 20); - - File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file"); - writeNBytesToFile(temp3, 75); - - FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10); - - assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - assertFalse(temp1.exists() || temp2.exists() || temp3.exists()); - } - - @Test - public void testDeleteDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2")); - - //23 files in root - // + 6 in test_folder1 + test_folder1 itself - assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteDirectoriesBasedOnAge() throws IOException { - initSubDirectories(); - // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is - // also older than maxAge inside the sub directory, create some files which are newer than maxAge. - // deleteDirectories() should NOT delete folder3 - File subFolder3 = folder.newFolder("test_folder3"); - File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3"); - - for (int j=0; j<11; j++) { - File.createTempFile("test_", ".json", subSubFolder3); - } - - subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*")); - - //23 files in root - // + 13 in test_folder2 - // + 13 in subSubFolder2 - // + 11 in subSubFolder3 - // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself - assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testRecursivelyDeleteDirectory() throws IOException { - initSubDirectories(); - FileHelper.recursiveDelete(folder.getRoot().toPath()); - assertFalse(folder.getRoot().exists()); - } - - @Test - public void testRecursivelyDeleteRegularFile() throws IOException { - File file = folder.newFile(); - assertTrue(file.exists()); - assertTrue(file.isFile()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testRecursivelyDeleteNonExistingFile() throws IOException { - File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile(); - assertFalse(file.exists()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testInitSubDirectories() throws IOException { - initSubDirectories(); - assertTrue(folder.getRoot().exists()); - assertTrue(folder.getRoot().isDirectory()); - - Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1"); - assertTrue(test_folder1.toFile().exists()); - assertTrue(test_folder1.toFile().isDirectory()); - - Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2"); - assertTrue(test_folder2.toFile().exists()); - assertTrue(test_folder2.toFile().isDirectory()); - - Path subSubFolder2 = test_folder2.resolve("subSubFolder2"); - assertTrue(subSubFolder2.toFile().exists()); - assertTrue(subSubFolder2.toFile().isDirectory()); - } - - @Test - public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException { - Path symPath = folder.getRoot().toPath().resolve("symlink"); - Path fakePath = Paths.get("/some/not/existant/file"); - - Files.createSymbolicLink(symPath, fakePath); - assertTrue(Files.isSymbolicLink(symPath)); - assertFalse(Files.exists(fakePath)); - - // Not possible to set modified time on symlink in java, so just check that it doesn't crash - FileHelper.getLastModifiedTime(symPath).toInstant(); - } - - private void initSubDirectories() throws IOException { - File subFolder1 = folder.newFolder("test_folder1"); - File subFolder2 = folder.newFolder("test_folder2"); - File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2"); - - for (int j=0; j<6; j++) { - File temp = File.createTempFile("test_", ".json", subFolder1); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis()); - } - - for (int j=0; j<9; j++) { - File.createTempFile("test_", ".json", subFolder2); - } - - for (int j=0; j<4; j++) { - File.createTempFile("abc_", ".txt", subFolder2); - } - - for (int j=0; j<13; j++) { - File temp = File.createTempFile("test_", ".json", subSubFolder2); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis()); - } - - //Must be after all the files have been created - subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis()); - subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - } - - private static int getNumberOfFilesAndDirectoriesIn(File folder) { - int total = 0; - for (File file : getContentsOfDirectory(folder)) { - if (file.isDirectory()) { - total += getNumberOfFilesAndDirectoriesIn(file); - } - total++; - } - - return total; - } - - private static void writeNBytesToFile(File file, int nBytes) throws IOException { - Files.write(file.toPath(), new byte[nBytes]); - } - - private static File[] getContentsOfDirectory(File directory) { - File[] directoryContents = directory.listFiles(); - - return directoryContents == null ? new File[0] : directoryContents; - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 8c3e3e28777..b0e1632002b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -291,7 +291,8 @@ public class NodeRepository extends AbstractComponent { Optional<Node> existing = getNode(node.hostname()); if (existing.isPresent()) throw new IllegalArgumentException("Cannot add " + node.hostname() + ": A node with this name already exists (" + - node + ", " + node.history() + ")"); + existing.get() + ", " + existing.get().history() + "). Node to be added: " + + node + ", " + node.history()); } try (Mutex lock = lockUnallocated()) { return db.addNodesInState(nodes, Node.State.reserved); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index f594e9bef9a..e437badf0dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -75,7 +75,7 @@ public class CapacityPolicies { if (canFail && nodeCount == 1 && Arrays.asList(ClusterSpec.Type.container, ClusterSpec.Type.content).contains(clusterType) && - zone.environment().isProduction() && zone.system() != SystemName.cd) + zone.environment().isProduction()) throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy"); return nodeCount; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index fd28975bfad..f8067cb8661 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; @@ -73,8 +74,9 @@ public class NodeRepositoryProvisioner implements Provisioner { throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + wantedGroups + " groups, " + "which doesn't allow the nodes to be divided evenly into groups"); - log.log(LogLevel.DEBUG, () -> "Received deploy prepare request for " + requestedCapacity + " in " + - wantedGroups + " groups for application " + application + ", cluster " + cluster); + log.log(zone.system() == SystemName.cd ? Level.INFO : LogLevel.DEBUG, + () -> "Received deploy prepare request for " + requestedCapacity + " in " + + wantedGroups + " groups for application " + application + ", cluster " + cluster); int effectiveGroups; NodeSpec requestedNodes; diff --git a/vespa-documentgen-plugin/etc/complex/book.sd b/vespa-documentgen-plugin/etc/complex/book.sd index 16bf4447979..6d6145cb10b 100644 --- a/vespa-documentgen-plugin/etc/complex/book.sd +++ b/vespa-documentgen-plugin/etc/complex/book.sd @@ -27,14 +27,12 @@ search book { field mywsfloat type weightedset<float> { indexing: attribute - attribute: prefetch } field mynestedwsfloat type weightedset<float> {} field myarrayint type array<int> { indexing: attribute - attribute: prefetch } field stringmap type map<string, string> { diff --git a/vespa-documentgen-plugin/etc/localapp/book.sd b/vespa-documentgen-plugin/etc/localapp/book.sd index 1f5e0e2cccb..72faa934aa6 100644 --- a/vespa-documentgen-plugin/etc/localapp/book.sd +++ b/vespa-documentgen-plugin/etc/localapp/book.sd @@ -27,12 +27,10 @@ search book { field mywsfloat type weightedset<float> { indexing: attribute - attribute: prefetch } field myarrayint type array<int> { indexing: attribute - attribute: prefetch } field mytriplearray type array<array<array<int>>> {} diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp index 569cce6db7b..d8ee3ea89ab 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp @@ -174,7 +174,7 @@ ssize_t CryptoCodecAdapter::half_close() { auto flush_res = flush_all(); - if ((flush_res < 0)) { + if (flush_res < 0) { return flush_res; } if (!_encoded_tls_close) { @@ -188,7 +188,7 @@ CryptoCodecAdapter::half_close() _encoded_tls_close = true; } flush_res = flush_all(); - if ((flush_res < 0)) { + if (flush_res < 0) { return flush_res; } return _socket.half_close(); |