From 99f13b90d34c9e3be47fd1859241fee068fdfa1b Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 3 Jan 2019 16:16:05 +0100 Subject: [PATCH] SNAPSHOT: Speed up HDFS Repository Writes (#37069) * There is no point in hsyncing after every individual write since there is only value in completely written blobs for restores, this is ensures by the `SYNC` flag already and there is no need for separately invoking `hsync` after individual writes --- .../repositories/hdfs/HdfsBlobContainer.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index 5a2e727d30b..e9b45a9b52e 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -97,19 +97,13 @@ final class HdfsBlobContainer extends AbstractBlobContainer { store.execute((Operation) fileContext -> { Path blob = new Path(path, blobName); // we pass CREATE, which means it fails if a blob already exists. - EnumSet flags = failIfAlreadyExists ? EnumSet.of(CreateFlag.CREATE, CreateFlag.SYNC_BLOCK) : - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK); - CreateOpts[] opts = {CreateOpts.bufferSize(bufferSize)}; - try (FSDataOutputStream stream = fileContext.create(blob, flags, opts)) { + EnumSet flags = failIfAlreadyExists ? EnumSet.of(CreateFlag.CREATE, CreateFlag.SYNC_BLOCK) + : EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK); + try (FSDataOutputStream stream = fileContext.create(blob, flags, CreateOpts.bufferSize(bufferSize))) { int bytesRead; byte[] buffer = new byte[bufferSize]; while ((bytesRead = inputStream.read(buffer)) != -1) { stream.write(buffer, 0, bytesRead); - // For safety we also hsync each write as well, because of its docs: - // SYNC_BLOCK - to force closed blocks to the disk device - // "In addition Syncable.hsync() should be called after each write, - // if true synchronous behavior is required" - stream.hsync(); } } catch (org.apache.hadoop.fs.FileAlreadyExistsException faee) { throw new FileAlreadyExistsException(blob.toString(), null, faee.getMessage());