367175 - SSL 100% CPU spin in case of blocked write and RST.

This commit is contained in:
Simone Bordet 2011-12-20 13:25:55 +01:00
parent 070a2b5759
commit ecb88f836e
3 changed files with 171 additions and 93 deletions

View File

@ -1,7 +1,5 @@
package org.eclipse.jetty.client;
import static org.hamcrest.Matchers.*;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
@ -18,17 +16,18 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLSocket;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.eclipse.jetty.http.HttpParser;
import org.eclipse.jetty.io.AsyncEndPoint;
import org.eclipse.jetty.io.Buffer;
import org.eclipse.jetty.io.Buffers;
import org.eclipse.jetty.io.Connection;
import org.eclipse.jetty.io.EndPoint;
@ -47,9 +46,13 @@ import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.lessThan;
public class SslBytesServerTest extends SslBytesTest
{
private final AtomicInteger sslHandles = new AtomicInteger();
private final AtomicInteger sslFlushes = new AtomicInteger();
private final AtomicInteger httpParses = new AtomicInteger();
private ExecutorService threadPool;
private Server server;
@ -75,6 +78,20 @@ public class SslBytesServerTest extends SslBytesTest
sslHandles.incrementAndGet();
return super.handle();
}
@Override
protected SslEndPoint newSslEndPoint()
{
return new SslEndPoint()
{
@Override
public int flush(Buffer buffer) throws IOException
{
sslFlushes.incrementAndGet();
return super.flush(buffer);
}
};
}
};
}
@ -99,7 +116,7 @@ public class SslBytesServerTest extends SslBytesTest
};
}
};
connector.setMaxIdleTime(2000);
connector.setMaxIdleTime(5000);
// connector.setPort(5870);
connector.setPort(0);
@ -119,9 +136,15 @@ public class SslBytesServerTest extends SslBytesTest
if (contentLength != null)
{
int length = Integer.parseInt(contentLength);
ServletInputStream input = request.getInputStream();
ServletInputStream input = httpRequest.getInputStream();
ServletOutputStream output = httpResponse.getOutputStream();
byte[] buffer = new byte[32 * 1024];
for (int i = 0; i < length; ++i)
input.read();
{
int read = input.read(buffer);
if ("/echo".equals(target))
output.write(buffer, 0, read);
}
}
}
});
@ -199,6 +222,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
closeClient(client);
@ -225,10 +249,8 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
byte[] chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
// Server Hello + Certificate + Server Done
record = proxy.readFromServer();
@ -241,10 +263,8 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
// Change Cipher Spec
record = proxy.readFromClient();
@ -253,10 +273,8 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
// Client Done
record = proxy.readFromClient();
@ -265,10 +283,8 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
// Change Cipher Spec
record = proxy.readFromServer();
@ -284,6 +300,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
client.close();
@ -295,10 +312,8 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
// Socket close
record = proxy.readFromClient();
Assert.assertNull(String.valueOf(record), record);
@ -358,6 +373,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
closeClient(client);
@ -380,10 +396,7 @@ public class SslBytesServerTest extends SslBytesTest
// Client Hello
TLSRecord record = proxy.readFromClient();
for (byte b : record.getBytes())
{
proxy.flushToServer(b);
TimeUnit.MILLISECONDS.sleep(50);
}
proxy.flushToServer(50, b);
// Server Hello + Certificate + Server Done
record = proxy.readFromServer();
@ -392,26 +405,17 @@ public class SslBytesServerTest extends SslBytesTest
// Client Key Exchange
record = proxy.readFromClient();
for (byte b : record.getBytes())
{
proxy.flushToServer(b);
TimeUnit.MILLISECONDS.sleep(50);
}
proxy.flushToServer(50, b);
// Change Cipher Spec
record = proxy.readFromClient();
for (byte b : record.getBytes())
{
proxy.flushToServer(b);
TimeUnit.MILLISECONDS.sleep(50);
}
proxy.flushToServer(50, b);
// Client Done
record = proxy.readFromClient();
for (byte b : record.getBytes())
{
proxy.flushToServer(b);
TimeUnit.MILLISECONDS.sleep(50);
}
proxy.flushToServer(50, b);
// Change Cipher Spec
record = proxy.readFromServer();
@ -440,10 +444,7 @@ public class SslBytesServerTest extends SslBytesTest
// Application data
record = proxy.readFromClient();
for (byte b : record.getBytes())
{
proxy.flushToServer(b);
TimeUnit.MILLISECONDS.sleep(50);
}
proxy.flushToServer(50, b);
Assert.assertNull(request.get(5, TimeUnit.SECONDS));
// Application data
@ -463,6 +464,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(750));
Assert.assertThat(sslFlushes.get(), lessThan(750));
Assert.assertThat(httpParses.get(), lessThan(150));
client.close();
@ -470,10 +472,7 @@ public class SslBytesServerTest extends SslBytesTest
// Close Alert
record = proxy.readFromClient();
for (byte b : record.getBytes())
{
proxy.flushToServer(b);
TimeUnit.MILLISECONDS.sleep(50);
}
proxy.flushToServer(50, b);
// Socket close
record = proxy.readFromClient();
Assert.assertNull(String.valueOf(record), record);
@ -587,6 +586,7 @@ public class SslBytesServerTest extends SslBytesTest
// Application data
TLSRecord record = proxy.readFromClient();
Assert.assertEquals(TLSRecord.Type.APPLICATION, record.getType());
proxy.flushToServer(record);
Assert.assertNull(request.get(5, TimeUnit.SECONDS));
@ -594,6 +594,7 @@ public class SslBytesServerTest extends SslBytesTest
// Close Alert
record = proxy.readFromClient();
Assert.assertEquals(TLSRecord.Type.ALERT, record.getType());
proxy.flushToServer(record);
// Do not close the raw socket yet
@ -614,6 +615,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
// Socket close
@ -652,6 +654,7 @@ public class SslBytesServerTest extends SslBytesTest
// Application data
TLSRecord record = proxy.readFromClient();
Assert.assertEquals(TLSRecord.Type.APPLICATION, record.getType());
proxy.flushToServer(record);
Assert.assertNull(request.get(5, TimeUnit.SECONDS));
@ -670,6 +673,64 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
client.close();
}
@Test
public void testRequestWithBigContentWriteBlockedAndResetException() throws Exception
{
final SSLSocket client = newClient();
SimpleProxy.AutomaticFlow automaticProxyFlow = proxy.startAutomaticFlow();
client.startHandshake();
Assert.assertTrue(automaticProxyFlow.stop(5, TimeUnit.SECONDS));
byte[] data = new byte[128 * 1024];
Arrays.fill(data, (byte)'X');
final String content = new String(data, "UTF-8");
Future<Object> request = threadPool.submit(new Callable<Object>()
{
public Object call() throws Exception
{
OutputStream clientOutput = client.getOutputStream();
clientOutput.write(("" +
"GET /echo HTTP/1.1\r\n" +
"Host: localhost\r\n" +
"Content-Length: " + content.length() + "\r\n" +
"\r\n" +
content).getBytes("UTF-8"));
clientOutput.flush();
return null;
}
});
// Nine TLSRecords will be generated for the request
for (int i = 0; i < 9; ++i)
{
// Application data
TLSRecord record = proxy.readFromClient();
Assert.assertEquals(TLSRecord.Type.APPLICATION, record.getType());
proxy.flushToServer(record, 0);
}
Assert.assertNull(request.get(5, TimeUnit.SECONDS));
// We asked the server to echo back the data we sent
// but we do not read it, thus causing a write interest
// on the server.
// However, we then simulate that the client resets the
// connection, and this will cause an exception in the
// server that is trying to write the data
proxy.resetServer();
// Wait a while to detect spinning
TimeUnit.SECONDS.sleep(1);
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
client.close();
@ -713,13 +774,11 @@ public class SslBytesServerTest extends SslBytesTest
byte[] bytes = new byte[dataBytes.length + closeBytes.length / 2];
System.arraycopy(dataBytes, 0, bytes, 0, dataBytes.length);
System.arraycopy(closeBytes, 0, bytes, dataBytes.length, closeBytes.length / 2);
proxy.flushToServer(bytes);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, bytes);
bytes = new byte[closeBytes.length - closeBytes.length / 2];
System.arraycopy(closeBytes, closeBytes.length / 2, bytes, 0, bytes.length);
proxy.flushToServer(bytes);
proxy.flushToServer(100, bytes);
// Do not close the raw socket yet
@ -739,6 +798,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
// Socket close
@ -785,13 +845,11 @@ public class SslBytesServerTest extends SslBytesTest
Assert.assertNull(request.get(5, TimeUnit.SECONDS));
byte[] chunk1 = new byte[2 * record.getBytes().length / 3];
System.arraycopy(record.getBytes(), 0, chunk1, 0, chunk1.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
byte[] chunk2 = new byte[record.getBytes().length - chunk1.length];
System.arraycopy(record.getBytes(), chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk2);
proxy.flushToServer(100, chunk2);
record = proxy.readFromServer();
Assert.assertEquals(TLSRecord.Type.APPLICATION, record.getType());
@ -809,6 +867,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
closeClient(client);
@ -855,14 +914,13 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
byte[] chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
}
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(150));
Assert.assertNull(request.get(5, TimeUnit.SECONDS));
@ -883,6 +941,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(150));
closeClient(client);
@ -1013,6 +1072,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(50));
closeClient(client);
@ -1068,10 +1128,8 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
byte[] chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
// Renegotiation Handshake
record = proxy.readFromServer();
@ -1108,10 +1166,8 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(chunk2);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
proxy.flushToServer(100, chunk2);
// Renegotiation Handshake
record = proxy.readFromClient();
@ -1121,8 +1177,7 @@ public class SslBytesServerTest extends SslBytesTest
System.arraycopy(bytes, 0, chunk1, 0, chunk1.length);
chunk2 = new byte[bytes.length - chunk1.length];
System.arraycopy(bytes, chunk1.length, chunk2, 0, chunk2.length);
proxy.flushToServer(chunk1);
TimeUnit.MILLISECONDS.sleep(100);
proxy.flushToServer(100, chunk1);
// Do not write the second chunk now, but merge it with content, see below
Assert.assertNull(renegotiation.get(5, TimeUnit.SECONDS));
@ -1146,7 +1201,7 @@ public class SslBytesServerTest extends SslBytesTest
byte[] mergedBytes = new byte[chunk2.length + dataBytes.length];
System.arraycopy(chunk2, 0, mergedBytes, 0, chunk2.length);
System.arraycopy(dataBytes, 0, mergedBytes, chunk2.length, dataBytes.length);
proxy.flushToServer(mergedBytes);
proxy.flushToServer(100, mergedBytes);
// Write the remaining 2 TLS records
for (int i = 0; i < 2; ++i)
{
@ -1176,6 +1231,7 @@ public class SslBytesServerTest extends SslBytesTest
// Check that we did not spin
Assert.assertThat(sslHandles.get(), lessThan(20));
Assert.assertThat(sslFlushes.get(), lessThan(20));
Assert.assertThat(httpParses.get(), lessThan(100));
closeClient(client);
@ -1251,7 +1307,7 @@ public class SslBytesServerTest extends SslBytesTest
return client;
}
private void closeClient(SSLSocket client) throws IOException
private void closeClient(SSLSocket client) throws Exception
{
client.close();
@ -1271,5 +1327,4 @@ public class SslBytesServerTest extends SslBytesTest
Assert.assertNull(String.valueOf(record), record);
proxy.flushToClient(record);
}
}

View File

@ -199,7 +199,12 @@ public abstract class SslBytesTest
return new TLSRecord(type, bytes);
}
public void flushToServer(TLSRecord record) throws IOException
public void flushToServer(TLSRecord record) throws Exception
{
flushToServer(record, 100);
}
public void flushToServer(TLSRecord record, long sleep) throws Exception
{
if (record == null)
{
@ -212,20 +217,22 @@ public abstract class SslBytesTest
}
else
{
flush(server, record.getBytes());
flush(sleep, server, record.getBytes());
}
}
public void flushToServer(byte... bytes) throws IOException
public void flushToServer(long sleep, byte... bytes) throws Exception
{
flush(server, bytes);
flush(sleep, server, bytes);
}
private void flush(Socket socket, byte... bytes) throws IOException
private void flush(long sleep, Socket socket, byte... bytes) throws Exception
{
OutputStream output = socket.getOutputStream();
output.write(bytes);
output.flush();
if (sleep > 0)
TimeUnit.MILLISECONDS.sleep(sleep);
}
public TLSRecord readFromServer() throws IOException
@ -235,7 +242,7 @@ public abstract class SslBytesTest
return record;
}
public void flushToClient(TLSRecord record) throws IOException
public void flushToClient(TLSRecord record) throws Exception
{
if (record == null)
{
@ -248,7 +255,7 @@ public abstract class SslBytesTest
}
else
{
flush(client, record.getBytes());
flush(0, client, record.getBytes());
}
}
@ -266,7 +273,7 @@ public abstract class SslBytesTest
{
while (true)
{
flushToServer(readFromClient());
flushToServer(readFromClient(), 0);
}
}
catch (InterruptedIOException x)
@ -313,6 +320,15 @@ public abstract class SslBytesTest
return latch.await(time, unit);
}
public void resetServer() throws IOException
{
// Calling setSoLinger(true, 0) causes close()
// to send a RST instead of a FIN, causing an
// exception to be thrown on the other end
server.setSoLinger(true, 0);
server.close();
}
public class AutomaticFlow
{
private final CountDownLatch stopLatch;

View File

@ -50,7 +50,7 @@ public class SslConnection extends AbstractConnection implements AsyncConnection
private final SSLEngine _engine;
private final SSLSession _session;
private AsyncConnection _connection;
private final SslEndPoint _sslEndPoint = new SslEndPoint();
private final SslEndPoint _sslEndPoint;
private int _allocations;
private SslBuffers _buffers;
private NIOBuffer _inbound;
@ -93,6 +93,12 @@ public class SslConnection extends AbstractConnection implements AsyncConnection
_engine=engine;
_session=_engine.getSession();
_aEndp=(AsyncEndPoint)endp;
_sslEndPoint = newSslEndPoint();
}
protected SslEndPoint newSslEndPoint()
{
return new SslEndPoint();
}
/* ------------------------------------------------------------ */
@ -308,12 +314,15 @@ public class SslConnection extends AbstractConnection implements AsyncConnection
if (_outbound.hasContent() && (flushed=_endp.flush(_outbound))>0)
progress = true;
}
catch (Exception e)
catch (IOException e)
{
LOG.debug(e.toString());
LOG.ignore(e);
_endp.close();
throw e;
}
finally
{
LOG.debug("{} {} {} filled={}/{} flushed={}/{}",_session,this,_engine.getHandshakeStatus(),filled,_inbound.length(),flushed,_outbound.length());
}
LOG.debug("{} {} {} filled={}/{} flushed={}/{}",_session,this,_engine.getHandshakeStatus(),filled,_inbound.length(),flushed,_outbound.length());
// handle the current hand share status
switch(_engine.getHandshakeStatus())
@ -435,7 +444,7 @@ public class SslConnection extends AbstractConnection implements AsyncConnection
}
catch(SSLException e)
{
LOG.warn(_endp+":",e);
LOG.warn(String.valueOf(_endp), e);
_endp.close();
throw e;
}
@ -511,10 +520,8 @@ public class SslConnection extends AbstractConnection implements AsyncConnection
}
catch(SSLException e)
{
LOG.warn(_endp+":"+e);
LOG.debug(e);
if (_endp.isOpen())
_endp.close();
LOG.warn(String.valueOf(_endp), e);
_endp.close();
throw e;
}
finally