diff options
45 files changed, 586 insertions, 125 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java index 950d2df9532..8d3e37e1ebd 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java @@ -87,9 +87,5 @@ public class IdentityDocumentGenerator { } } - private static String toZoneDnsSuffix(Zone zone, String dnsSuffix) { - return zone.environment().value() + "-" + zone.region().value() + "." + dnsSuffix; - } - } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 55373f425e0..883156ab533 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -54,7 +54,6 @@ public interface ModelContext { boolean isFirstTimeDeployment(); boolean useDedicatedNodeForLogserver(); boolean useFdispatchByDefault(); - boolean dispatchWithProtobuf(); boolean useAdaptiveDispatch(); boolean enableMetricsProxyContainer(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 4b35af53154..19eb4ba19f6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -35,7 +35,6 @@ public class TestProperties implements ModelContext.Properties { private boolean isFirstTimeDeployment = false; private boolean useDedicatedNodeForLogserver = false; private boolean useFdispatchByDefault = true; - private boolean dispatchWithProtobuf = true; private boolean useAdaptiveDispatch = false; private boolean enableMetricsProxyContainer = false; @@ -54,7 +53,6 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public boolean useFdispatchByDefault() { return useFdispatchByDefault; } - @Override public boolean dispatchWithProtobuf() { return dispatchWithProtobuf; } @Override public boolean enableMetricsProxyContainer() { return enableMetricsProxyContainer; } public TestProperties setApplicationId(ApplicationId applicationId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index ee82d0cd719..9caf7fbdc9e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -107,7 +107,6 @@ public class IndexedSearchCluster extends SearchCluster private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; private final boolean useFdispatchByDefault; - private final boolean dispatchWithProtobuf; private final boolean useAdaptiveDispatch; private List<SearchNode> searchNodes = new ArrayList<>(); @@ -127,7 +126,6 @@ public class IndexedSearchCluster extends SearchCluster dispatchParent = new SimpleConfigProducer(this, "dispatchers"); rootDispatch = new DispatchGroup(this); useFdispatchByDefault = deployState.getProperties().useFdispatchByDefault(); - dispatchWithProtobuf = deployState.getProperties().dispatchWithProtobuf(); useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch(); } @@ -439,7 +437,6 @@ public class IndexedSearchCluster extends SearchCluster builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); builder.useMultilevelDispatch(useMultilevelDispatchSetup()); builder.useFdispatchByDefault(useFdispatchByDefault); - builder.dispatchWithProtobuf(dispatchWithProtobuf); builder.useLocalNode(tuning.dispatch.useLocalNode); builder.searchableCopies(rootDispatch.getSearchableCopies()); if (searchCoverage != null) { diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index c3f847fb5f3..892b72c8081 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -19,9 +19,6 @@ distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN # Should fdispatch be used as the default dispatcher useFdispatchByDefault bool default=true -# Should protobuf/jrt be preferred over fs4 -dispatchWithProtobuf bool default=false - # Is multi-level dispatch configured for this cluster useMultilevelDispatch bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 0279d175488..e75726f5a19 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -131,7 +131,6 @@ public class ModelContextImpl implements ModelContext { private final boolean useDedicatedNodeForLogserver; private final boolean useFdispatchByDefault; private final boolean useAdaptiveDispatch; - private final boolean dispatchWithProtobuf; private final boolean enableMetricsProxyContainer; public Properties(ApplicationId applicationId, @@ -161,8 +160,6 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useFdispatchByDefault = Flags.USE_FDISPATCH_BY_DEFAULT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.dispatchWithProtobuf = Flags.DISPATCH_WITH_PROTOBUF.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.enableMetricsProxyContainer = Flags.ENABLE_METRICS_PROXY_CONTAINER.bindTo(flagSource) @@ -213,9 +210,6 @@ public class ModelContextImpl implements ModelContext { public boolean useFdispatchByDefault() { return useFdispatchByDefault; } @Override - public boolean dispatchWithProtobuf() { return dispatchWithProtobuf; } - - @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index a63f709a99f..87daf1181da 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -111,13 +111,10 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { return; // global config access ok } else { String hostname = configRequest.getClientHostName(); - Optional<RequestHandler> tenantHandler = - Optional.ofNullable(hostRegistry.getKeyForHost(hostname)) - .flatMap(this::getTenantHandler); - if (tenantHandler.isEmpty()) { - return; // unknown host - } - ApplicationId resolvedApplication = tenantHandler.get().resolveApplicationId(hostname); + TenantName tenantName = Optional.ofNullable(hostRegistry.getKeyForHost(hostname)) + .orElseThrow(() -> new AuthorizationException(String.format("Host '%s' not found in host registry", hostname))); + RequestHandler tenantHandler = getTenantHandler(tenantName); + ApplicationId resolvedApplication = tenantHandler.resolveApplicationId(hostname); ApplicationId peerOwner = applicationId(peerIdentity); if (peerOwner.equals(resolvedApplication)) { return; // allowed to access @@ -141,11 +138,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { case host: ApplicationId peerOwner = applicationId(peerIdentity); FileReference requestedFile = new FileReference(request.parameters().get(0).asString()); - RequestHandler tenantHandler = getTenantHandler(peerOwner.tenant()) - .orElseThrow(() -> new AuthorizationException( - String.format( - "Application '%s' does not exist - unable to verify file ownership for '%s'", - peerOwner.toShortString(), requestedFile.value()))); + RequestHandler tenantHandler = getTenantHandler(peerOwner.tenant()); Set<FileReference> filesOwnedByApplication = tenantHandler.listFileReferences(peerOwner); if (filesOwnedByApplication.contains(requestedFile)) { return; // allowed to access @@ -182,7 +175,9 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { throw new IllegalStateException("Client authentication is not enforced!"); // clients should be required to authenticate when TLS is enabled } try { - return Optional.of(nodeIdentifier.identifyNode(certChain)); + NodeIdentity identity = nodeIdentifier.identifyNode(certChain); + log.log(LogLevel.DEBUG, () -> String.format("Client '%s' identified as %s", request.target().toString(), identity.toString())); + return Optional.of(identity); } catch (NodeIdentifierException e) { throw new AuthorizationException("Failed to identity peer: " + e.getMessage(), e); } @@ -194,11 +189,12 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { private static ApplicationId applicationId(NodeIdentity peerIdentity) { return peerIdentity.applicationId() - .orElseThrow(() -> new AuthorizationException("Peer node is not associated with an application")); + .orElseThrow(() -> new AuthorizationException("Peer node is not associated with an application: " + peerIdentity.toString())); } - private Optional<RequestHandler> getTenantHandler(TenantName tenantName) { - return handlerProvider.getRequestHandler(tenantName); + private RequestHandler getTenantHandler(TenantName tenantName) { + return handlerProvider.getRequestHandler(tenantName) + .orElseThrow(() -> new AuthorizationException(String.format("No handler exists for tenant '%s'", tenantName.value()))); } @SuppressWarnings("unchecked") diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java index d8d916e567e..b0f4bd3a4ee 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java @@ -174,6 +174,46 @@ public class MultiTenantRpcAuthorizerTest { .get(); } + @Test + public void tenant_node_must_be_registered_in_host_registry() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(EVIL_APP_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request configRequest = createConfigRequest(new ConfigKey<>("name", "configid", "namespace"), HOSTNAME); + + exceptionRule.expectMessage("Host 'myhostname' not found in host registry"); + exceptionRule.expectCause(instanceOf(AuthorizationException.class)); + + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + @Test + public void tenant_must_have_a_request_handler() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(EVIL_APP_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + hostRegistry.update(EVIL_APP_ID.tenant(), List.of(HOSTNAME.value())); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request configRequest = createConfigRequest(new ConfigKey<>("name", "configid", "namespace"), HOSTNAME); + + exceptionRule.expectMessage("No handler exists for tenant 'malice'"); + exceptionRule.expectCause(instanceOf(AuthorizationException.class)); + + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + private static RpcAuthorizer createAuthorizer(NodeIdentity identity, HostRegistry<TenantName> hostRegistry) { return new MultiTenantRpcAuthorizer( new StaticNodeIdentifier(identity), @@ -199,6 +239,7 @@ public class MultiTenantRpcAuthorizerTest { RequestHandlerProvider handlerProvider = mock(RequestHandlerProvider.class); when(handlerProvider.getRequestHandler(APPLICATION_ID.tenant())).thenReturn(Optional.of(requestHandler)); + when(handlerProvider.getRequestHandler(EVIL_APP_ID.tenant())).thenReturn(Optional.empty()); return handlerProvider; } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java index 3a597a34142..ffce3649419 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java @@ -7,8 +7,7 @@ import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.DatatypeFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated @@ -18,9 +17,7 @@ public class DatatypeFactoryProvider implements Provider<DatatypeFactory> { @Override public DatatypeFactory get() { try { - return DatatypeFactory.newInstance( - FACTORY_CLASS, - this.getClass().getClassLoader()); + return DatatypeFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); } catch (DatatypeConfigurationException e) { throw new IllegalStateException(e); } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java index 10d707337b5..37b8dff8bf4 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java @@ -6,20 +6,20 @@ import com.yahoo.container.di.componentgraph.Provider; import javax.xml.parsers.DocumentBuilderFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class DocumentBuilderFactoryProvider implements Provider<DocumentBuilderFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"; @Override public DocumentBuilderFactory get() { - return DocumentBuilderFactory.newInstance(FACTORY_CLASS, - this.getClass().getClassLoader()); + return DocumentBuilderFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); } @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java index 2ef44fed3eb..a8ac55a8aca 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java @@ -6,20 +6,20 @@ import com.yahoo.container.di.componentgraph.Provider; import javax.xml.parsers.SAXParserFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class SAXParserFactoryProvider implements Provider<SAXParserFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"; @Override public SAXParserFactory get() { - return SAXParserFactory.newInstance(FACTORY_CLASS, - this.getClass().getClassLoader()); + return SAXParserFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); } @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java index ec024f60909..bbcdf7c9553 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java @@ -7,21 +7,20 @@ import javax.xml.XMLConstants; import javax.xml.validation.SchemaFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class SchemaFactoryProvider implements Provider<SchemaFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.validation.XMLSchemaFactory"; @Override public SchemaFactory get() { - return SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI, - FACTORY_CLASS, - this.getClass().getClassLoader()); + return SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI, FACTORY_CLASS, this.getClass().getClassLoader()); } @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java index fad3fbd964a..974d2e6a259 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java @@ -6,20 +6,20 @@ import com.yahoo.container.di.componentgraph.Provider; import javax.xml.transform.TransformerFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class TransformerFactoryProvider implements Provider<TransformerFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xalan.internal.xsltc.trax.TransformerFactoryImpl"; @Override public TransformerFactory get() { - return TransformerFactory.newInstance(FACTORY_CLASS, - this.getClass().getClassLoader()); + return TransformerFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); } @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java index e894c39865b..702ecedcc93 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java @@ -6,12 +6,12 @@ import com.yahoo.container.di.componentgraph.Provider; import javax.xml.stream.XMLEventFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class XMLEventFactoryProvider implements Provider<XMLEventFactory> { + public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.events.XMLEventFactoryImpl"; @Override @@ -23,4 +23,5 @@ public class XMLEventFactoryProvider implements Provider<XMLEventFactory> { @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java index 8bdef13fbcc..9f3518525de 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java @@ -6,12 +6,12 @@ import com.yahoo.container.di.componentgraph.Provider; import javax.xml.stream.XMLInputFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class XMLInputFactoryProvider implements Provider<XMLInputFactory> { + private static final String INPUT_FACTORY_INTERFACE = XMLInputFactory.class.getName(); public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.XMLInputFactoryImpl"; @@ -28,4 +28,5 @@ public class XMLInputFactoryProvider implements Provider<XMLInputFactory> { @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java index ac0715b6b88..ab28ba2e923 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java @@ -6,12 +6,12 @@ import com.yahoo.container.di.componentgraph.Provider; import javax.xml.stream.XMLOutputFactory; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class XMLOutputFactoryProvider implements Provider<XMLOutputFactory> { + public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.XMLOutputFactoryImpl"; @Override public XMLOutputFactory get() { @@ -24,4 +24,5 @@ public class XMLOutputFactoryProvider implements Provider<XMLOutputFactory> { @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java index 7c6837d9b97..407369032cf 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java @@ -7,12 +7,12 @@ import javax.xml.xpath.XPathFactory; import javax.xml.xpath.XPathFactoryConfigurationException; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 + * @author Einar M R Rosenvinge * @deprecated Do not use! */ @Deprecated public class XPathFactoryProvider implements Provider<XPathFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xpath.internal.jaxp.XPathFactoryImpl"; @Override @@ -28,4 +28,5 @@ public class XPathFactoryProvider implements Provider<XPathFactory> { @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java index d9af9bf88b2..ace5a7ab304 100644 --- a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java +++ b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java @@ -14,12 +14,12 @@ import com.yahoo.language.opennlp.OpenNlpLinguistics; * * @author bratseth */ +@SuppressWarnings("unused") // Injected public class DefaultLinguisticsProvider implements Provider<Linguistics> { // Use lazy initialization to avoid expensive (memory-wise) instantiation f private volatile Supplier<Linguistics> linguisticsSupplier = Suppliers.memoize(OpenNlpLinguistics::new); - @SuppressWarnings("deprecation") @Inject public DefaultLinguisticsProvider() { } diff --git a/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java b/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java index 7e01505dc03..3996dff2811 100644 --- a/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java +++ b/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java @@ -306,6 +306,15 @@ public class ContainerTest extends ContainerTestBase { assertTrue(destructableEntity.deconstructed); } + @Test + public void providers_are_invoked_only_when_needed() { + writeBootstrapConfigs("id1", FailOnGetProvider.class); + + Container container = newContainer(dirConfigSource); + + ComponentGraph oldGraph = container.getNewComponentGraph(); + } + static class DestructableEntity { private boolean deconstructed = false; } @@ -323,6 +332,18 @@ public class ContainerTest extends ContainerTestBase { } } + public static class FailOnGetProvider implements Provider<Integer> { + + public Integer get() { + fail("Should never be called."); + return null; + } + + public void deconstruct() { + } + + } + public static class ComponentTakingConfig extends AbstractComponent { private final TestConfig config; diff --git a/container-di/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-di/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index 337c875b429..a5934bcf098 100644 --- a/container-di/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-di/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -39,6 +39,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -51,6 +52,7 @@ import static org.junit.Assert.fail; * @author ollivir */ public class ComponentGraphTest { + public static class ConfigMap extends HashMap<ConfigKey<? extends ConfigInstance>, ConfigInstance> { public ConfigMap() { super(); @@ -240,7 +242,8 @@ public class ComponentGraphTest { Integer instance1 = componentGraph.getInstance(Integer.class); Integer instance2 = componentGraph.getInstance(Integer.class); - assertThat(instance1, not(equalTo(instance2))); + assertEquals(1, instance1.intValue()); + assertEquals(2, instance2.intValue()); } @Test @@ -266,7 +269,7 @@ public class ComponentGraphTest { componentGraph.add(mockComponentNode(ComponentTakingExecutor.class)); componentGraph.add(mockComponentNode(ExecutorProvider.class)); - componentGraph.add(mockComponentNode(IntProvider.class)); + componentGraph.add(mockComponentNode(FailOnGetIntProvider.class)); componentGraph.complete(); assertNotNull(componentGraph.getInstance(ComponentTakingExecutor.class)); @@ -559,21 +562,23 @@ public class ComponentGraphTest { public static class DerivedExecutorProvider extends ExecutorProvider { } - public static class IntProvider implements Provider<Integer> { + public static class FailOnGetIntProvider implements Provider<Integer> { + public Integer get() { - throw new AssertionError("Should never be called."); + fail("Should never be called."); + return null; } public void deconstruct() { } + } public static class NewIntProvider implements Provider<Integer> { int i = 0; public Integer get() { - i++; - return i; + return ++i; } public void deconstruct() { @@ -590,6 +595,7 @@ public class ComponentGraphTest { } public static class ComponentWithInjectConstructor { + public ComponentWithInjectConstructor(TestConfig c, Test2Config c2) { throw new RuntimeException("Should not be called"); } @@ -597,9 +603,11 @@ public class ComponentGraphTest { @Inject public ComponentWithInjectConstructor(Test2Config c) { } + } public static class ComponentWithMultipleConstructors { + private ComponentWithMultipleConstructors(int dummy) { } @@ -615,6 +623,7 @@ public class ComponentGraphTest { public ComponentWithMultipleConstructors(Test2Config c) { this(); } + } public static class ComponentTakingComponentId { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java index 1a08b569b4d..a3f8819f843 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java @@ -15,6 +15,7 @@ import com.yahoo.metrics.MetricsPresentationConfig; * * @author bratseth */ +@SuppressWarnings("unused") // Injected public class MetricConsumerProviderProvider implements Provider<MetricConsumerProvider> { private final MetricConsumerProvider provided; @@ -30,7 +31,6 @@ public class MetricConsumerProviderProvider implements Provider<MetricConsumerPr public MetricConsumerProvider get() { return provided; } @Override - public void deconstruct() { - } + public void deconstruct() { } } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java index 57613113e73..d18e1ee2d5d 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java @@ -32,8 +32,6 @@ public final class MetricProvider implements Provider<Metric> { } @Override - public void deconstruct() { - - } + public void deconstruct() { } } diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java index 1dbe410ba54..b8c714fd3e3 100644 --- a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java +++ b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java @@ -32,7 +32,9 @@ import static com.yahoo.container.servlet.jersey.util.ResourceConfigUtil.registe * @author Tony Vaagenes * @author ollivir */ +@SuppressWarnings("unused") // Injected public class JerseyServletProvider implements Provider<ServletHolder> { + private final ServletHolder jerseyServletHolder; public JerseyServletProvider(RestApiContext restApiContext) { @@ -40,7 +42,7 @@ public class JerseyServletProvider implements Provider<ServletHolder> { } private ResourceConfig resourceConfig(RestApiContext restApiContext) { - final ResourceConfig resourceConfig = ResourceConfig + ResourceConfig resourceConfig = ResourceConfig .forApplication(new JerseyApplication(resourcesAndProviders(restApiContext.getBundles()))); registerComponent(resourceConfig, componentInjectorBinder(restApiContext)); @@ -51,7 +53,7 @@ public class JerseyServletProvider implements Provider<ServletHolder> { } private static Collection<Class<?>> resourcesAndProviders(Collection<BundleInfo> bundles) { - final List<Class<?>> ret = new ArrayList<>(); + List<Class<?>> ret = new ArrayList<>(); for (BundleInfo bundle : bundles) { for (String classEntry : bundle.getClassEntries()) { @@ -115,4 +117,5 @@ public class JerseyServletProvider implements Provider<ServletHolder> { @Override public void deconstruct() { } + } diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java index 89d4aa05200..b90f96fb240 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java @@ -14,7 +14,7 @@ import com.yahoo.messagebus.shared.SharedSourceSession; /** * @author Tony Vaagenes - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class MbusClientProvider implements Provider<MbusClient> { diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java index 58f0df6ddcf..1ab418fd523 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java @@ -18,6 +18,7 @@ import java.util.logging.Logger; * @author Tony Vaagenes */ public class MbusServerProvider implements Provider<MbusServer> { + private static final Logger log = Logger.getLogger(MbusServerProvider.class.getName()); private final MbusServer server; @@ -52,4 +53,5 @@ public class MbusServerProvider implements Provider<MbusServer> { server.release(); sessionRef.getReference().close(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 1754b6cc028..2f241f9c7a3 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -171,7 +171,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } } - private void doVisit(final long timestamp, final Object payload, final boolean hasChildren) throws IOException { + private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException { boolean dirty = false; if (timestamp != 0L) { header(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 68e84c7d289..891a696be9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -692,7 +692,9 @@ public class InternalStepRunner implements StepRunner { " </filtering>\n" + " </http>\n" + "\n" + - " <nodes count=\"1\" flavor=\"" + flavor + "\" allocated-memory=\"" + jdiscMemoryPercentage + "%\" />\n" + + " <nodes count=\"1\" flavor=\"" + flavor + "\">\n" + + " <jvm allocated-memory=\"" + jdiscMemoryPercentage + "%\" />\n" + + " </nodes>\n" + " </container>\n" + "</services>\n"; diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index df276f9bbbc..bd49f3be55a 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -38,6 +38,8 @@ </filtering> </http> - <nodes count="1" flavor="d-2-12-75" allocated-memory="17%" /> + <nodes count="1" flavor="d-2-12-75"> + <jvm allocated-memory="17%" /> + </nodes> </container> </services> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9ede09efec4..ad2654b67ae 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -93,12 +93,6 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); - public static final UnboundBooleanFlag DISPATCH_WITH_PROTOBUF = defineFeatureFlag( - "dispatch-with-protobuf", false, - "Should the java dispatcher use protobuf/jrt as the default", - "Takes effect at redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_DYNAMIC_PROVISIONING = defineFeatureFlag( "enable-dynamic-provisioning", false, "Provision a new docker host when we otherwise can't allocate a docker node", diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java deleted file mode 100644 index 810eb5fb908..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java +++ /dev/null @@ -1,14 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import java.util.List; - -/** - * @author Unknown - */ -public interface ServiceListener { - void setServices(List<VespaService> services); -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index e19d65f2838..e33674e87b0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -26,11 +26,11 @@ import java.util.logging.Logger; * * @author Eirik Nygaard */ -public class SystemPoller implements ServiceListener { +public class SystemPoller { final private static Logger log = Logger.getLogger(SystemPoller.class.getName()); private final int pollingIntervalSecs; - private volatile List<VespaService> services; + private final List<VespaService> services; private final int memoryTypeVirtual = 0; private final int memoryTypeResident = 1; @@ -45,12 +45,6 @@ public class SystemPoller implements ServiceListener { systemPollTimer = new Timer("systemPollTimer", true); } - @Override - public void setServices(List<VespaService> services) { - log.log(LogLevel.DEBUG, "Setting services in SystemPoller to: " + services); - this.services = services; - } - void stop() { systemPollTimer.cancel(); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java index dea7b2b4809..30fd79c1338 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java @@ -30,4 +30,5 @@ public class SystemPollerProvider implements Provider<SystemPoller> { public SystemPoller get() { return poller; } + } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java index a42d52b7ea6..c5f2ea6960b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java @@ -25,8 +25,7 @@ public class SystemPollerTest { SystemPoller poller = new SystemPoller(services, 60*5); assertThat(s.isAlive(), is(false)); - services.remove(0); - poller.setServices(services); + long n = poller.getPidJiffies(s); assertThat(n, is(0L)); long[] memusage = poller.getMemoryUsage(s); diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java index 4c8890f6476..767af147ad7 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java @@ -3,6 +3,12 @@ package ai.vespa.rankingexpression.importer.vespa; import ai.vespa.rankingexpression.importer.ImportedModel; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.tensor.Tensor; import org.junit.Test; import java.util.List; @@ -32,14 +38,17 @@ public class VespaImportTestCase { assertEquals("tensor(x[3]):{{x:0}:0.5,{x:1}:1.5,{x:2}:2.5}", model.largeConstants().get("constant1asLarge")); assertEquals(2, model.expressions().size()); - assertEquals("max(reduce(input1 * input2, sum, name) * constant1,x) * constant2", + assertEquals("reduce(reduce(input1 * input2, sum, name) * constant1, max, x) * constant2", model.expressions().get("foo1").getRoot().toString()); - assertEquals("max(reduce(input1 * input2, sum, name) * constant1asLarge,x) * constant2", + assertEquals("reduce(reduce(input1 * input2, sum, name) * constant1asLarge, max, x) * constant2", model.expressions().get("foo2").getRoot().toString()); List<ImportedMlFunction> functions = model.outputExpressions(); assertEquals(2, functions.size()); ImportedMlFunction foo1Function = functions.get(0); + assertEquals("foo1", foo1Function.name()); + assertEquals("reduce(reduce(input1 * input2, sum, name) * constant1, max, x) * constant2", foo1Function.expression()); + assertEquals("tensor():{202.5}", evaluate(foo1Function, "{{name:a, x:0}: 1, {name:a, x:1}: 2, {name:a, x:2}: 3}").toString()); assertEquals(2, foo1Function.arguments().size()); assertTrue(foo1Function.arguments().contains("input1")); assertTrue(foo1Function.arguments().contains("input2")); @@ -80,4 +89,17 @@ public class VespaImportTestCase { return model; } + private Tensor evaluate(ImportedMlFunction function, String input1Argument) { + try { + MapContext context = new MapContext(); + context.put("input1", new TensorValue(Tensor.from(function.argumentTypes().get("input1"), input1Argument))); + context.put("input2", new TensorValue(Tensor.from(function.argumentTypes().get("input2"), "{{x:0}:3, {x:1}:6, {x:2}:9}"))); + context.put("constant1", new TensorValue(Tensor.from("tensor(x[3]):{{x:0}:0.5, {x:1}:1.5, {x:2}:2.5}"))); + context.put("constant2", new TensorValue(Tensor.from("tensor():{{}:3}"))); + return new RankingExpression(function.expression()).evaluate(context).asTensor(); + } + catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } } diff --git a/model-integration/src/test/models/vespa/example.model b/model-integration/src/test/models/vespa/example.model index 9579be4e44c..66d21cfc53f 100644 --- a/model-integration/src/test/models/vespa/example.model +++ b/model-integration/src/test/models/vespa/example.model @@ -15,11 +15,11 @@ model example { } function foo1() { - expression: max(sum(input1 * input2, name) * constant1, x) * constant2 + expression: reduce(sum(input1 * input2, name) * constant1, max, x) * constant2 } function foo2() { - expression: max(sum(input1 * input2, name) * constant1asLarge, x) * constant2 + expression: reduce(sum(input1 * input2, name) * constant1asLarge, max, x) * constant2 } }
\ No newline at end of file diff --git a/security-tools/pom.xml b/security-tools/pom.xml index aa07e96d628..76882927059 100644 --- a/security-tools/pom.xml +++ b/security-tools/pom.xml @@ -19,6 +19,12 @@ <version>${project.version}</version> <scope>compile</scope> </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>1.4</version> + <scope>compile</scope> + </dependency> <!-- test scope --> <dependency> @@ -27,6 +33,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/CliOptions.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/CliOptions.java new file mode 100644 index 00000000000..f3ec73236c4 --- /dev/null +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/CliOptions.java @@ -0,0 +1,69 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.security.tool.securityenv; + +import com.yahoo.security.tls.TransportSecurityUtils; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.Arrays; + +import static java.util.stream.Collectors.joining; + +/** + * Defines the program's command line parameters. + * + * @author bjorncs + */ +class CliOptions { + static final String SHELL_OPTION = "shell"; + static final String HELP_OPTION = "help"; + + private static final Options OPTIONS = new Options() + .addOption( + Option.builder("s") + .longOpt(SHELL_OPTION) + .hasArg(true) + .required(false) + .desc(String.format("Shell type. Shell type is auto-detected if option not present. Valid values: %s.", + Arrays.stream(UnixShell.values()) + .map(shell -> String.format("'%s'", shell.configName())) + .collect(joining(", ", "[", "]")))) + .build()) + .addOption(Option.builder("h") + .longOpt(HELP_OPTION) + .hasArg(false) + .required(false) + .desc("Show help") + .build()); + + static CommandLine parseCliArguments(String[] cliArgs) throws ParseException { + CommandLineParser parser = new DefaultParser(); + return parser.parse(OPTIONS, cliArgs); + } + + static void printHelp(PrintStream out) { + HelpFormatter formatter = new HelpFormatter(); + PrintWriter writer = new PrintWriter(out); + formatter.printHelp( + writer, + formatter.getWidth(), + "vespa-security-env <options>", + String.format("Generates shell commands that defines environments variables based on the content of %s.", + TransportSecurityUtils.CONFIG_FILE_ENVIRONMENT_VARIABLE), + OPTIONS, + formatter.getLeftPadding(), + formatter.getDescPadding(), + String.format("The output may include the following variables:\n%s\n", + Arrays.stream(OutputVariable.values()) + .map(variable -> String.format(" - '%s': %s", variable.variableName(), variable.description())) + .collect(joining("\n")))); + writer.flush(); + } +} diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java index f57575b406a..74c08c2d602 100644 --- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java @@ -1,11 +1,87 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.security.tool.securityenv; +import com.yahoo.security.tls.MixedMode; +import com.yahoo.security.tls.TransportSecurityOptions; +import com.yahoo.security.tls.TransportSecurityUtils; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.ParseException; + +import java.io.PrintStream; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +import static com.yahoo.vespa.security.tool.securityenv.CliOptions.HELP_OPTION; +import static com.yahoo.vespa.security.tool.securityenv.CliOptions.SHELL_OPTION; + /** + * Implementation of the 'vespa-security-env' command line utility. + * * @author bjorncs */ public class Main { + + private final PrintStream stdOut; + private final PrintStream stdError; + + Main(PrintStream stdOut, PrintStream stdError) { + this.stdOut = stdOut; + this.stdError = stdError; + } + public static void main(String[] args) { - System.out.println("TODO implementation"); + Main program = new Main(System.out, System.err); + int statusCode = program.execute(args, System.getenv()); + System.exit(statusCode); + } + + int execute(String[] cliArgs, Map<String, String> envVars) { + boolean debugMode = envVars.containsKey("VESPA_DEBUG"); + try { + CommandLine arguments = CliOptions.parseCliArguments(cliArgs); + if (arguments.hasOption(HELP_OPTION)) { + CliOptions.printHelp(stdOut); + return 0; + } + UnixShell shell = arguments.hasOption(SHELL_OPTION) + ? UnixShell.fromConfigName(arguments.getOptionValue(SHELL_OPTION)) + : UnixShell.detect(envVars.get("SHELL")); + + Optional<TransportSecurityOptions> options = TransportSecurityUtils.getOptions(envVars); + if (options.isEmpty()) { + return 0; + } + Map<String, String> outputVariables = new TreeMap<>(); + options.get().getCaCertificatesFile() + .ifPresent(caCertFile -> addOutputVariable(outputVariables, OutputVariable.CA_CERTIFICATE, caCertFile.toString())); + MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(envVars); + if (mixedMode != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) { + options.get().getCertificatesFile() + .ifPresent(certificateFile -> addOutputVariable(outputVariables, OutputVariable.CERTIFICATE, certificateFile.toString())); + options.get().getPrivateKeyFile() + .ifPresent(privateKeyFile -> addOutputVariable(outputVariables, OutputVariable.PRIVATE_KEY, privateKeyFile.toString())); + } + shell.writeOutputVariables(stdOut, outputVariables); + return 0; + } catch (ParseException e) { + return handleException("Failed to parse command line arguments: " + e.getMessage(), e, debugMode); + } catch (IllegalArgumentException e) { + return handleException("Invalid command line arguments: " + e.getMessage(), e, debugMode); + } catch (Exception e) { + return handleException("Failed to generate security environment variables: " + e.getMessage(), e, debugMode); + } + } + + private static void addOutputVariable(Map<String, String> outputVariables, OutputVariable variable, String value) { + outputVariables.put(variable.variableName(), value); + } + + private int handleException(String message, Exception exception, boolean debugMode) { + stdError.println(message); + if (debugMode) { + exception.printStackTrace(stdError); + } + return 1; } } diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java new file mode 100644 index 00000000000..9cd4cc1fc67 --- /dev/null +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java @@ -0,0 +1,29 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.security.tool.securityenv; + +/** + * Define the possible environment variables that the program may output. + * + * @author bjorncs + */ +enum OutputVariable { + CA_CERTIFICATE("VESPA_TLS_CA_CERT", "Path to CA certificates file"), + CERTIFICATE("VESPA_TLS_CERT", "Path to certificate file"), + PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file"); + + private final String variableName; + private final String description; + + OutputVariable(String variableName, String description) { + this.variableName = variableName; + this.description = description; + } + + String variableName() { + return variableName; + } + + String description() { + return description; + } +} diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/UnixShell.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/UnixShell.java new file mode 100644 index 00000000000..74630b4e1f4 --- /dev/null +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/UnixShell.java @@ -0,0 +1,74 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.security.tool.securityenv; + +import java.io.PrintStream; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Definition of some unix shell variants and how to export environments variable for those supported. + * The output format is inspired by ssh-agent's output. + * + * @author bjorncs + */ +enum UnixShell { + BOURNE("bourne", List.of("bash", "sh")) { + @Override + void writeOutputVariables(PrintStream out, Map<String, String> outputVariables) { + outputVariables.forEach((name, value) -> { + out.print(name); + out.print("=\""); + out.print(value); // note: value is assumed to need no escaping + out.print("\"; export "); + out.print(name); + out.println(';'); + }); + } + }, + CSHELL("cshell", List.of("csh", "fish")) { + @Override + void writeOutputVariables(PrintStream out, Map<String, String> outputVariables) { + outputVariables.forEach((name, value) -> { + out.print("setenv "); + out.print(name); + out.print(" \""); + out.print(value); // note: value is assumed to need no escaping + out.println("\";"); + }); + } + }; + + private static final UnixShell DEFAULT = BOURNE; + + private final String configName; + private final List<String> knownShellBinaries; + + UnixShell(String configName, List<String> knownShellBinaries) { + this.configName = configName; + this.knownShellBinaries = knownShellBinaries; + } + + abstract void writeOutputVariables(PrintStream out, Map<String, String> outputVariables); + + String configName() { + return configName; + } + + static UnixShell fromConfigName(String configName) { + return Arrays.stream(values()) + .filter(shell -> shell.configName.equals(configName)) + .findAny() + .orElseThrow(() -> new IllegalArgumentException("Unknown shell: " + configName)); + } + + static UnixShell detect(String shellEnvVariable) { + if (shellEnvVariable == null || shellEnvVariable.isEmpty()) return DEFAULT; + int lastSlash = shellEnvVariable.lastIndexOf('/'); + String shellName = lastSlash != -1 ? shellEnvVariable.substring(lastSlash + 1) : shellEnvVariable; + return Arrays.stream(values()) + .filter(shell -> shell.knownShellBinaries.contains(shellName)) + .findAny() + .orElse(DEFAULT); + } +} diff --git a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java new file mode 100644 index 00000000000..6b25c2a2bce --- /dev/null +++ b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java @@ -0,0 +1,114 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.security.tool.securityenv; + +import com.yahoo.security.tls.MixedMode; +import com.yahoo.security.tls.TransportSecurityOptions; +import com.yahoo.security.tls.TransportSecurityUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + + +/** + * @author bjorncs + */ +public class MainTest { + + private final ByteArrayOutputStream stdOutBytes = new ByteArrayOutputStream(); + private final ByteArrayOutputStream stdErrBytes = new ByteArrayOutputStream(); + private final PrintStream stdOut = new PrintStream(stdOutBytes); + private final PrintStream stdError = new PrintStream(stdErrBytes); + + @Rule + public TemporaryFolder tmpFolder = new TemporaryFolder(); + + @Test + public void prints_help_page_on_help_option() throws IOException { + int exitCode = runMain(List.of("--help"), Map.of()); + assertThat(exitCode).isEqualTo(0); + assertThat(stdOut()).isEqualTo(readTestResource("expected-help-output.txt")); + } + + @Test + public void prints_no_output_when_no_security_config() { + int exitCode = runMain(List.of(), Map.of()); + assertThat(exitCode).isEqualTo(0); + assertThat(stdErr()).isEmpty(); + } + + @Test + public void prints_security_variables_with_specified_shell() throws IOException { + Path configFile = generateConfigFile(); + Map<String, String> env = Map.of(TransportSecurityUtils.CONFIG_FILE_ENVIRONMENT_VARIABLE, configFile.toString()); + int exitCode = runMain(List.of(), env); + assertThat(exitCode).isEqualTo(0); + assertThat(stdOut()).isEqualTo(readTestResource("bash-output.txt")); + } + + @Test + public void prints_security_variables_with_auto_detected_shell() throws IOException { + Path configFile = generateConfigFile(); + Map<String, String> env = Map.of( + TransportSecurityUtils.CONFIG_FILE_ENVIRONMENT_VARIABLE, configFile.toString(), + TransportSecurityUtils.INSECURE_MIXED_MODE_ENVIRONMENT_VARIABLE, MixedMode.TLS_CLIENT_MIXED_SERVER.configValue(), + "SHELL", "/usr/local/bin/fish"); + int exitCode = runMain(List.of(), env); + assertThat(exitCode).isEqualTo(0); + assertThat(stdOut()).isEqualTo(readTestResource("csh-output.txt")); + } + + + @Test + public void prints_error_message_on_unknown_shell_name() { + int exitCode = runMain(List.of("--shell", "invalid-shell-name"), Map.of()); + assertThat(exitCode).isEqualTo(1); + assertThat(stdErr()).isEqualTo("Invalid command line arguments: Unknown shell: invalid-shell-name\n"); + } + + @Test + public void prints_error_message_on_unknown_command_line_parameter() { + int exitCode = runMain(List.of("--unknown-parameter"), Map.of()); + assertThat(exitCode).isEqualTo(1); + assertThat(stdErr()).isEqualTo("Failed to parse command line arguments: Unrecognized option: --unknown-parameter\n"); + } + + private int runMain(List<String> args, Map<String, String> env) { + return new Main(stdOut, stdError).execute(args.toArray(new String[0]), env); + } + + private String stdOut() { + stdOut.flush(); + return stdOutBytes.toString(); + } + + private String stdErr() { + stdError.flush(); + return stdErrBytes.toString(); + } + + private static String readTestResource(String fileName) throws IOException { + return Files.readString(Paths.get(MainTest.class.getResource('/' + fileName).getFile())); + } + + private Path generateConfigFile() throws IOException { + TransportSecurityOptions options = new TransportSecurityOptions.Builder() + .withCertificates(Paths.get("/path/to/certificate"), Paths.get("/path/to/key")) + .withCaCertificates(Paths.get("/path/to/cacerts")) + .build(); + Path configFile = tmpFolder.newFile().toPath(); + options.toJsonFile(configFile); + return configFile; + } + +}
\ No newline at end of file diff --git a/security-tools/src/test/resources/bash-output.txt b/security-tools/src/test/resources/bash-output.txt new file mode 100644 index 00000000000..421320f82d5 --- /dev/null +++ b/security-tools/src/test/resources/bash-output.txt @@ -0,0 +1,3 @@ +VESPA_TLS_CA_CERT="/path/to/cacerts"; export VESPA_TLS_CA_CERT; +VESPA_TLS_CERT="/path/to/certificate"; export VESPA_TLS_CERT; +VESPA_TLS_PRIVATE_KEY="/path/to/key"; export VESPA_TLS_PRIVATE_KEY; diff --git a/security-tools/src/test/resources/csh-output.txt b/security-tools/src/test/resources/csh-output.txt new file mode 100644 index 00000000000..47594af9339 --- /dev/null +++ b/security-tools/src/test/resources/csh-output.txt @@ -0,0 +1,3 @@ +setenv VESPA_TLS_CA_CERT "/path/to/cacerts"; +setenv VESPA_TLS_CERT "/path/to/certificate"; +setenv VESPA_TLS_PRIVATE_KEY "/path/to/key"; diff --git a/security-tools/src/test/resources/expected-help-output.txt b/security-tools/src/test/resources/expected-help-output.txt new file mode 100644 index 00000000000..e16f1b1dab0 --- /dev/null +++ b/security-tools/src/test/resources/expected-help-output.txt @@ -0,0 +1,10 @@ +usage: vespa-security-env <options> +Generates shell commands that defines environments variables based on the +content of VESPA_TLS_CONFIG_FILE. + -h,--help Show help + -s,--shell <arg> Shell type. Shell type is auto-detected if option not + present. Valid values: ['bourne', 'cshell']. +The output may include the following variables: + - 'VESPA_TLS_CA_CERT': Path to CA certificates file + - 'VESPA_TLS_CERT': Path to certificate file + - 'VESPA_TLS_PRIVATE_KEY': Path to private key file diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java index f5f9182fc4e..a4e508e0d2a 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java @@ -3,6 +3,7 @@ package com.yahoo.security.tls; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Map; import java.util.Optional; /** @@ -19,27 +20,47 @@ public class TransportSecurityUtils { private TransportSecurityUtils() {} public static boolean isTransportSecurityEnabled() { - return getConfigFile().isPresent(); + return isTransportSecurityEnabled(System.getenv()); + } + + public static boolean isTransportSecurityEnabled(Map<String, String> envVariables) { + return getConfigFile(envVariables).isPresent(); } public static MixedMode getInsecureMixedMode() { - return getEnvironmentVariable(INSECURE_MIXED_MODE_ENVIRONMENT_VARIABLE) + return getInsecureMixedMode(System.getenv()); + } + + public static MixedMode getInsecureMixedMode(Map<String, String> envVariables) { + return getEnvironmentVariable(envVariables, INSECURE_MIXED_MODE_ENVIRONMENT_VARIABLE) .map(MixedMode::fromConfigValue) .orElse(MixedMode.defaultValue()); } public static AuthorizationMode getInsecureAuthorizationMode() { - return getEnvironmentVariable(INSECURE_AUTHORIZATION_MODE_ENVIRONMENT_VARIABLE) + return getInsecureAuthorizationMode(System.getenv()); + } + + public static AuthorizationMode getInsecureAuthorizationMode(Map<String, String> envVariables) { + return getEnvironmentVariable(envVariables, INSECURE_AUTHORIZATION_MODE_ENVIRONMENT_VARIABLE) .map(AuthorizationMode::fromConfigValue) .orElse(AuthorizationMode.defaultValue()); } public static Optional<Path> getConfigFile() { - return getEnvironmentVariable(CONFIG_FILE_ENVIRONMENT_VARIABLE).map(Paths::get); + return getConfigFile(System.getenv()); + } + + public static Optional<Path> getConfigFile(Map<String, String> envVariables) { + return getEnvironmentVariable(envVariables, CONFIG_FILE_ENVIRONMENT_VARIABLE).map(Paths::get); } public static Optional<TransportSecurityOptions> getOptions() { - return getConfigFile() + return getOptions(System.getenv()); + } + + public static Optional<TransportSecurityOptions> getOptions(Map<String, String> envVariables) { + return getConfigFile(envVariables) .map(TransportSecurityOptions::fromJsonFile); } @@ -48,8 +69,8 @@ public class TransportSecurityUtils { .map(configFile -> new ReloadingTlsContext(configFile, getInsecureAuthorizationMode())); } - private static Optional<String> getEnvironmentVariable(String environmentVariable) { - return Optional.ofNullable(System.getenv(environmentVariable)) + private static Optional<String> getEnvironmentVariable(Map<String, String> environmentVariables, String variableName) { + return Optional.ofNullable(environmentVariables.get(variableName)) .filter(var -> !var.isEmpty()); } } |