У меня есть вспомогательная процедура, которая пытается загружать потоки с S3. Очень часто (около 1% запросов) я получаю сообщение журнала о NoHttpResponseException
, которое через некоторое время вызывает SocketTimeoutException
при чтении из S3ObjectInputStream
.
Я делаю что-то не так, или это просто мой маршрутизатор/интернет? Или это можно ожидать от S3? Я не замечаю проблем в другом месте.
public void
fastRead(final String key, Path path) throws StorageException
{
final int pieceSize = 1<<20;
final int threadCount = 8;
try (FileChannel channel = (FileChannel) Files.newByteChannel( path, WRITE, CREATE, TRUNCATE_EXISTING ))
{
final long size = s3.getObjectMetadata(bucket, key).getContentLength();
final long pieceCount = (size - 1) / pieceSize + 1;
ThreadPool pool = new ThreadPool (threadCount);
final AtomicInteger progress = new AtomicInteger();
for(int i = 0; i < size; i += pieceSize)
{
final int start = i;
final long end = Math.min(i + pieceSize, size);
pool.submit(() ->
{
boolean retry;
do
{
retry = false;
try
{
GetObjectRequest request = new GetObjectRequest(bucket, key);
request.setRange(start, end - 1);
S3Object piece = s3.getObject(request);
ByteBuffer buffer = ByteBuffer.allocate ((int)(end - start));
try(InputStream stream = piece.getObjectContent())
{
IOUtils.readFully( stream, buffer.array() );
}
channel.write( buffer, start );
double percent = (double) progress.incrementAndGet() / pieceCount * 100.0;
System.err.printf("%.1f%%\n", percent);
}
catch(java.net.SocketTimeoutException | java.net.SocketException e)
{
System.err.println("Read timed out. Retrying...");
retry = true;
}
}
while (retry);
});
}
pool.<IOException>await();
}
catch(AmazonClientException | IOException | InterruptedException e)
{
throw new StorageException (e);
}
}
2014-05-28 08:49:58 INFO com.amazonaws.http.AmazonHttpClient executeHelper Unable to execute HTTP request: The target server failed to respond
org.apache.http.NoHttpResponseException: The target server failed to respond
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:95)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:62)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:254)
at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:289)
at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:252)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:191)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:300)
at com.amazonaws.http.protocol.SdkHttpRequestExecutor.doReceiveResponse(SdkHttpRequestExecutor.java:66)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:127)
at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:713)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:518)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:385)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:233)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3569)
at com.amazonaws.services.s3.AmazonS3Client.getObject(AmazonS3Client.java:1130)
at com.syncwords.files.S3Storage.lambda$fastRead$0(S3Storage.java:123)
at com.syncwords.files.S3Storage$$Lambda$3/1397088232.run(Unknown Source)
at net.almson.util.ThreadPool.lambda$submit$8(ThreadPool.java:61)
at net.almson.util.ThreadPool$$Lambda$4/1980698753.call(Unknown Source)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:744)