diff --git a/README.md b/README.md index 9a6adf84a96..42e165bec44 100644 --- a/README.md +++ b/README.md @@ -447,6 +447,36 @@ To run test: mvn -Dtests.azure=true -Dtests.config=/path/to/config/file/elasticsearch.yml clean test ``` +Optimizing index storage on Azure File Service +============================================== +When using a shared file system based on the SMB protocol (like Azure File Service) to store indices, the way Lucene open index segment files +can slow down some functionalities like indexing documents and recovering index shards. If you'd like to know more about this behavior and how Lucene and SMB interact, +have a look at [LUCENE-6176](https://issues.apache.org/jira/browse/LUCENE-6176). + +In order to get better performance, the Azure Cloud plugin provides two storage types optimized for SMB: + +- `org.elasticsearch.index.store.fs.SmbMmapFsIndexStoreModule`: a SMB specific implementation of the default [mmap fs](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/index-modules-store.html#mmapfs) +- `org.elasticsearch.index.store.fs.SmbSimpleFsIndexStoreModule`: a SMB specific implementation of the default [simple fs](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/index-modules-store.html#simplefs) + +To use one of these specific storage types, you need to install the Azure Cloud plugin and restart the node. Then configure Elasticsearch to set +the storage type you want. + +This can be configured for all indices by adding this to the config/elasticsearch.yml file: +``` +index.store.type: org.elasticsearch.index.store.fs.SmbSimpleFsIndexStoreModule +``` + +Note that setting will be applied for newly created indices. + +It can also be set on a per-index basis at index creation time: +``` +curl -XPUT localhost:9200/my_index -d '{ + "settings": { + "index.store.type": "org.elasticsearch.index.store.fs.SmbMmapFsIndexStoreModule" + } +}' +``` + License ------- diff --git a/src/main/java/org/apache/lucene/store/SmbDirectoryWrapper.java b/src/main/java/org/apache/lucene/store/SmbDirectoryWrapper.java new file mode 100644 index 00000000000..1e783fdd6c5 --- /dev/null +++ b/src/main/java/org/apache/lucene/store/SmbDirectoryWrapper.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.store; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; + +/** + * This class is used to wrap an existing {@link org.apache.lucene.store.FSDirectory} so that + * the new shard segment files will be opened for Read and Write access. + *

+ * When storing index files on an SMB share like Azure File Service, opening the file for Read + * access can save a lot of roundtrips to the storage server and thus offering better performance. + */ +public final class SmbDirectoryWrapper extends FilterDirectory { + + private final FSDirectory fsDirectory; + + public SmbDirectoryWrapper(FSDirectory in) { + super(in); + fsDirectory = in; + } + + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + fsDirectory.ensureOpen(); + fsDirectory.ensureCanWrite(name); + return new SmbFSIndexOutput(name); + } + + /** + * Copied from final inner class {@link org.apache.lucene.store.FSDirectory.FSIndexOutput} + */ + final class SmbFSIndexOutput extends OutputStreamIndexOutput { + /** + * The maximum chunk size is 8192 bytes, because {@link java.io.FileOutputStream} mallocs + * a native buffer outside of stack if the write buffer size is larger. + */ + static final int CHUNK_SIZE = 8192; + + private final String name; + + public SmbFSIndexOutput(String name) throws IOException { + super("SmbFSIndexOutput(path=\"" + fsDirectory.getDirectory().resolve(name) + "\")", new FilterOutputStream(Channels.newOutputStream(Files.newByteChannel(fsDirectory.getDirectory().resolve(name), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.READ, StandardOpenOption.WRITE))) { + // This implementation ensures, that we never write more than CHUNK_SIZE bytes: + @Override + public void write(byte[] b, int offset, int length) throws IOException { + while (length > 0) { + final int chunk = Math.min(length, CHUNK_SIZE); + out.write(b, offset, chunk); + length -= chunk; + offset += chunk; + } + } + }, CHUNK_SIZE); + this.name = name; + } + } +} diff --git a/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsDirectoryService.java b/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsDirectoryService.java new file mode 100644 index 00000000000..cbdfb3f527d --- /dev/null +++ b/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsDirectoryService.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.MMapDirectory; +import org.apache.lucene.store.SmbDirectoryWrapper; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.IndexStore; + +import java.io.IOException; +import java.nio.file.Path; + +public class SmbMmapFsDirectoryService extends FsDirectoryService { + + @Inject + public SmbMmapFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { + super(shardId, indexSettings, indexStore); + } + + @Override + protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { + logger.debug("wrapping MMapDirectory for SMB"); + return new SmbDirectoryWrapper(new MMapDirectory(location, buildLockFactory())); + } +} diff --git a/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsIndexStore.java b/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsIndexStore.java new file mode 100644 index 00000000000..76a7a04310c --- /dev/null +++ b/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsIndexStore.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.support.AbstractIndexStore; +import org.elasticsearch.indices.store.IndicesStore; + +public class SmbMmapFsIndexStore extends AbstractIndexStore { + + @Inject + public SmbMmapFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) { + super(index, indexSettings, indexService, indicesStore, nodeEnv); + } + + @Override + public Class shardDirectory() { + return SmbMmapFsDirectoryService.class; + } +} diff --git a/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsIndexStoreModule.java b/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsIndexStoreModule.java new file mode 100644 index 00000000000..768fab5c0db --- /dev/null +++ b/src/main/java/org/elasticsearch/index/store/fs/SmbMmapFsIndexStoreModule.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.index.store.IndexStore; + +public class SmbMmapFsIndexStoreModule extends AbstractModule { + + @Override + protected void configure() { + bind(IndexStore.class).to(SmbMmapFsIndexStore.class).asEagerSingleton(); + } +} diff --git a/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsDirectoryService.java b/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsDirectoryService.java new file mode 100644 index 00000000000..61020bc1ef3 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsDirectoryService.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.SmbDirectoryWrapper; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.IndexStore; + +import java.io.IOException; +import java.nio.file.Path; + +public class SmbSimpleFsDirectoryService extends FsDirectoryService { + + @Inject + public SmbSimpleFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { + super(shardId, indexSettings, indexStore); + } + + @Override + protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { + logger.debug("wrapping SimpleFSDirectory for SMB"); + return new SmbDirectoryWrapper(new SimpleFSDirectory(location, lockFactory)); + } +} diff --git a/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsIndexStore.java b/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsIndexStore.java new file mode 100644 index 00000000000..bd3a5ad539b --- /dev/null +++ b/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsIndexStore.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.support.AbstractIndexStore; +import org.elasticsearch.indices.store.IndicesStore; + +public class SmbSimpleFsIndexStore extends AbstractIndexStore { + + @Inject + public SmbSimpleFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) { + super(index, indexSettings, indexService, indicesStore, nodeEnv); + } + + @Override + public Class shardDirectory() { + return SmbSimpleFsDirectoryService.class; + } +} + diff --git a/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsIndexStoreModule.java b/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsIndexStoreModule.java new file mode 100644 index 00000000000..88d075ab77b --- /dev/null +++ b/src/main/java/org/elasticsearch/index/store/fs/SmbSimpleFsIndexStoreModule.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.index.store.IndexStore; + +public class SmbSimpleFsIndexStoreModule extends AbstractModule { + + @Override + protected void configure() { + bind(IndexStore.class).to(SmbSimpleFsIndexStore.class).asEagerSingleton(); + } +} diff --git a/src/test/java/org/apache/lucene/store/SmbMMapDirectoryTest.java b/src/test/java/org/apache/lucene/store/SmbMMapDirectoryTest.java new file mode 100644 index 00000000000..65d837f7760 --- /dev/null +++ b/src/test/java/org/apache/lucene/store/SmbMMapDirectoryTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.store; + +import java.io.IOException; +import java.nio.file.Path; + +public class SmbMMapDirectoryTest extends BaseDirectoryTestCase { + + @Override + protected Directory getDirectory(Path file) throws IOException { + return new SmbDirectoryWrapper(new MMapDirectory(file)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/lucene/store/SmbSimpleFSDirectoryTest.java b/src/test/java/org/apache/lucene/store/SmbSimpleFSDirectoryTest.java new file mode 100644 index 00000000000..5031411155c --- /dev/null +++ b/src/test/java/org/apache/lucene/store/SmbSimpleFSDirectoryTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.store; + +import java.io.IOException; +import java.nio.file.Path; + +public class SmbSimpleFSDirectoryTest extends BaseDirectoryTestCase { + + @Override + protected Directory getDirectory(Path file) throws IOException { + return new SmbDirectoryWrapper(new SimpleFSDirectory(file)); + } +}