NioSslEngine implements NioFilter { public boolean handshake(SocketChannel socketChannel, int timeout, ByteBuffer peerNetData) throws IOException, InterruptedException { if (peerNetData.capacity() < engine.getSession().getPacketBufferSize()) { throw new IllegalArgumentException(String.format("Provided buffer is too small to perform " + "SSL handshake. Buffer capacity is %s but need %s", peerNetData.capacity(), engine.getSession().getPacketBufferSize())); } ByteBuffer handshakeBuffer = peerNetData; handshakeBuffer.clear(); ByteBuffer myAppData = ByteBuffer.wrap(new byte[0]); if (logger.isDebugEnabled()) { logger.debug("Starting TLS handshake with {}. Timeout is {}ms", socketChannel.socket(), timeout); } long timeoutNanos = -1; if (timeout > 0) { timeoutNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout); } engine.beginHandshake(); SSLEngineResult.HandshakeStatus status = engine.getHandshakeStatus(); SSLEngineResult engineResult = null; while (status != FINISHED && status != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) { if (socketChannel.socket().isClosed()) { logger.info("Handshake terminated because socket is closed"); throw new SocketException("handshake terminated - socket is closed"); } if (timeoutNanos > 0) { if (timeoutNanos < System.nanoTime()) { logger.info("TLS handshake is timing out"); throw new SocketTimeoutException("handshake timed out"); } } switch (status) { case NEED_UNWRAP: int dataRead = socketChannel.read(handshakeBuffer); handshakeBuffer.flip(); engineResult = engine.unwrap(handshakeBuffer, peerAppData); handshakeBuffer.compact(); status = engineResult.getHandshakeStatus(); if (peerAppData.remaining() == 0 && dataRead == 0 && status == NEED_UNWRAP) { Thread.sleep(10); } if (engineResult.getStatus() == BUFFER_OVERFLOW) { peerAppData = expandWriteBuffer(TRACKED_RECEIVER, peerAppData, peerAppData.capacity() * 2); } break; case NEED_WRAP: myNetData.clear(); engineResult = engine.wrap(myAppData, myNetData); status = engineResult.getHandshakeStatus(); switch (engineResult.getStatus()) { case BUFFER_OVERFLOW: myNetData = expandWriteBuffer(TRACKED_SENDER, myNetData, myNetData.capacity() * 2); break; case OK: myNetData.flip(); while (myNetData.hasRemaining()) { socketChannel.write(myNetData); } break; case CLOSED: break; default: logger.info("handshake terminated with illegal state due to {}", status); throw new IllegalStateException( "Unknown SSLEngineResult status: " + engineResult.getStatus()); } break; case NEED_TASK: handleBlockingTasks(); status = engine.getHandshakeStatus(); break; default: logger.info("handshake terminated with illegal state due to {}", status); throw new IllegalStateException("Unknown SSL Handshake state: " + status); } Thread.sleep(10); } if (status != FINISHED) { logger.info("handshake terminated with exception due to {}", status); throw new SSLHandshakeException("SSL Handshake terminated with status " + status); } if (logger.isDebugEnabled()) { if (engineResult != null) { logger.debug("TLS handshake successful. result={} and handshakeResult={}", engineResult.getStatus(), engine.getHandshakeStatus()); } else { logger.debug("TLS handshake successful. handshakeResult={}", engine.getHandshakeStatus()); } } return true; } NioSslEngine(SSLEngine engine, BufferPool bufferPool); boolean handshake(SocketChannel socketChannel, int timeout,
      ByteBuffer peerNetData); @Override synchronized ByteBuffer wrap(ByteBuffer appData); @Override synchronized ByteBuffer unwrap(ByteBuffer wrappedBuffer); @Override ByteBuffer ensureWrappedCapacity(int amount, ByteBuffer wrappedBuffer,
      BufferType bufferType); @Override ByteBuffer readAtLeast(SocketChannel channel, int bytes,
      ByteBuffer wrappedBuffer); @Override ByteBuffer getUnwrappedBuffer(ByteBuffer wrappedBuffer); ByteBuffer ensureUnwrappedCapacity(int amount); @Override void doneReadingDirectAck(ByteBuffer unwrappedBuffer); @Override synchronized void close(SocketChannel socketChannel);  }
@Test public void handshake() throws Exception { SocketChannel mockChannel = mock(SocketChannel.class); when(mockChannel.read(any(ByteBuffer.class))).thenReturn(100, 100, 100, 0); Socket mockSocket = mock(Socket.class); when(mockChannel.socket()).thenReturn(mockSocket); when(mockSocket.isClosed()).thenReturn(false); when(mockEngine.getHandshakeStatus()).thenReturn(NEED_UNWRAP, NEED_WRAP); when(mockEngine.unwrap(any(ByteBuffer.class), any(ByteBuffer.class))).thenReturn( new SSLEngineResult(OK, NEED_WRAP, 100, 100), new SSLEngineResult(BUFFER_OVERFLOW, NEED_UNWRAP, 0, 0), new SSLEngineResult(OK, NEED_TASK, 100, 0)); when(mockEngine.getDelegatedTask()).thenReturn(() -> { }, (Runnable) null); when(mockEngine.wrap(any(ByteBuffer.class), any(ByteBuffer.class))).thenReturn( new SSLEngineResult(OK, NEED_UNWRAP, 100, 100), new SSLEngineResult(BUFFER_OVERFLOW, NEED_WRAP, 0, 0), new SSLEngineResult(CLOSED, FINISHED, 100, 0)); spyNioSslEngine.handshake(mockChannel, 10000, ByteBuffer.allocate(netBufferSize)); verify(mockEngine, atLeast(2)).getHandshakeStatus(); verify(mockEngine, times(3)).wrap(any(ByteBuffer.class), any(ByteBuffer.class)); verify(mockEngine, times(3)).unwrap(any(ByteBuffer.class), any(ByteBuffer.class)); verify(spyNioSslEngine, times(2)).expandWriteBuffer(any(BufferPool.BufferType.class), any(ByteBuffer.class), any(Integer.class)); verify(spyNioSslEngine, times(1)).handleBlockingTasks(); verify(mockChannel, times(3)).read(any(ByteBuffer.class)); }
@Test public void handshakeWithInsufficientBufferSize() throws Exception { SocketChannel mockChannel = mock(SocketChannel.class); when(mockChannel.read(any(ByteBuffer.class))).thenReturn(100, 100, 100, 0); Socket mockSocket = mock(Socket.class); when(mockChannel.socket()).thenReturn(mockSocket); when(mockSocket.isClosed()).thenReturn(false); assertThatThrownBy(() -> spyNioSslEngine.handshake(mockChannel, 10000, ByteBuffer.allocate(netBufferSize / 2))).isExactlyInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Provided buffer is too small"); }
@Test public void handshakeDetectsClosedSocket() throws Exception { SocketChannel mockChannel = mock(SocketChannel.class); when(mockChannel.read(any(ByteBuffer.class))).thenReturn(100, 100, 100, 0); Socket mockSocket = mock(Socket.class); when(mockChannel.socket()).thenReturn(mockSocket); when(mockSocket.isClosed()).thenReturn(true); when(mockEngine.getHandshakeStatus()).thenReturn(NEED_UNWRAP); ByteBuffer byteBuffer = ByteBuffer.allocate(netBufferSize); assertThatThrownBy(() -> spyNioSslEngine.handshake(mockChannel, 10000, byteBuffer)) .isInstanceOf( SocketException.class) .hasMessageContaining("handshake terminated"); }