Support TLSv1.3 in JRT

Allow reception of TLSv1.3 post-handshake session ticket:
1) Allow transition transition NOT_HANDSHAKING => FINISHED during application data wrap/unwrap.
2) Correctly handle unwrap where zero application data is unwrapped.

1 files changed, 22 insertions, 21 deletions

diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java
index 91dbfccb203..f140bb38c66 100644
--- a/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java
+++ b/jrt/src/com/yahoo/jrt/TlsCryptoSocket.java
@@ -6,6 +6,7 @@ import com.yahoo.security.tls.authz.PeerAuthorizerTrustManager;
 
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLEngineResult.HandshakeStatus;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.SSLPeerUnverifiedException;
@@ -21,7 +22,6 @@ import java.util.Optional;
 import java.util.logging.Logger;
 
 import static java.util.stream.Collectors.toList;
-import static javax.net.ssl.SSLEngineResult.HandshakeStatus;
 import static javax.net.ssl.SSLEngineResult.Status;
 
 /**
@@ -49,7 +49,6 @@ public class TlsCryptoSocket implements CryptoSocket {
     private AuthorizationResult authorizationResult;
 
     public TlsCryptoSocket(SocketChannel channel, SSLEngine sslEngine) {
-        disableTlsv13(sslEngine);
         this.channel = channel;
         this.sslEngine = sslEngine;
         SSLSession nullSession = sslEngine.getSession();
@@ -190,12 +189,11 @@ public class TlsCryptoSocket implements CryptoSocket {
     public int drain(ByteBuffer dst) throws IOException {
         verifyHandshakeCompleted();
         int totalBytesUnwrapped = 0;
-        int bytesUnwrapped;
-        do {
-            bytesUnwrapped = applicationDataUnwrap(dst);
-            totalBytesUnwrapped += bytesUnwrapped;
-        } while (bytesUnwrapped > 0);
-        return totalBytesUnwrapped;
+        while (true) {
+            int result = applicationDataUnwrap(dst);
+            if (result < 0) return totalBytesUnwrapped;
+            totalBytesUnwrapped += result;
+        }
     }
 
     @Override
@@ -250,7 +248,7 @@ public class TlsCryptoSocket implements CryptoSocket {
 
     private int applicationDataWrap(ByteBuffer src) throws IOException {
         SSLEngineResult result = sslEngineWrap(src);
-        if (result.getHandshakeStatus() != HandshakeStatus.NOT_HANDSHAKING) throw new SSLException("Renegotiation detected");
+        failIfRenegotiationDetected(result);
         switch (result.getStatus()) {
             case OK:
                 return result.bytesConsumed();
@@ -263,7 +261,7 @@ public class TlsCryptoSocket implements CryptoSocket {
 
     private SSLEngineResult sslEngineWrap(ByteBuffer src) throws IOException {
         SSLEngineResult result = sslEngine.wrap(src, wrapBuffer.getWritable(sessionPacketBufferSize));
-        if (result.getStatus() == Status.CLOSED) throw new ClosedChannelException();
+        failIfCloseSignalDetected(result);
         return result;
     }
 
@@ -282,13 +280,13 @@ public class TlsCryptoSocket implements CryptoSocket {
 
     private int applicationDataUnwrap(ByteBuffer dst) throws IOException {
         SSLEngineResult result = sslEngineUnwrap(dst);
-        if (result.getHandshakeStatus() != HandshakeStatus.NOT_HANDSHAKING) throw new SSLException("Renegotiation detected");
+        failIfRenegotiationDetected(result);
         switch (result.getStatus()) {
             case OK:
                 return result.bytesProduced();
             case BUFFER_OVERFLOW:
             case BUFFER_UNDERFLOW:
-                return 0;
+                return -1;
             default:
                 throw unexpectedStatusException(result.getStatus());
         }
@@ -296,7 +294,7 @@ public class TlsCryptoSocket implements CryptoSocket {
 
     private SSLEngineResult sslEngineUnwrap(ByteBuffer dst) throws IOException {
         SSLEngineResult result = sslEngine.unwrap(unwrapBuffer.getReadable(), dst);
-        if (result.getStatus() == Status.CLOSED) throw new ClosedChannelException();
+        failIfCloseSignalDetected(result);
         return result;
     }
 
@@ -312,6 +310,17 @@ public class TlsCryptoSocket implements CryptoSocket {
         return channel.write(wrapBuffer.getReadable());
     }
 
+    private static void failIfCloseSignalDetected(SSLEngineResult result) throws ClosedChannelException {
+        if (result.getStatus() == Status.CLOSED) throw new ClosedChannelException();
+    }
+
+    private static void failIfRenegotiationDetected(SSLEngineResult result) throws SSLException {
+        if (result.getHandshakeStatus() != HandshakeStatus.NOT_HANDSHAKING
+                && result.getHandshakeStatus() != HandshakeStatus.FINISHED) {
+            throw new SSLException("Renegotiation detected");
+        }
+    }
+
     private static IllegalStateException unhandledStateException(HandshakeState state) {
         return new IllegalStateException("Unhandled state: " + state);
     }
@@ -325,12 +334,4 @@ public class TlsCryptoSocket implements CryptoSocket {
             throw new SSLException("Handshake not completed: handshakeState=" + handshakeState);
     }
 
-    private static void disableTlsv13(SSLEngine sslEngine) {
-        String[] filteredProtocols = Arrays.stream(sslEngine.getEnabledProtocols())
-                .filter(p -> !p.equals("TLSv1.3"))
-                .toArray(String[]::new);
-        if (filteredProtocols.length == 0) throw new IllegalArgumentException("JRT does not support TLSv1.3");
-        sslEngine.setEnabledProtocols(filteredProtocols);
-    }
-
 }