honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16940 from s1monw/issues/16936 

Use SleepingWrapper on shared filesytems
This commit is contained in:
Simon Willnauer 2016-03-03 21:52:22 +01:00
parent 6adbbff97c 6a76fa3382
commit 4504a58643
4 changed files with 81 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
View File

@ -29,8 +29,10 @@ import org.apache.lucene.store.NativeFSLockFactory;
import org.apache.lucene.store.RateLimitedFSDirectory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.lucene.store.SleepingLockWrapper;
import org.apache.lucene.store.StoreRateLimiting;
import org.apache.lucene.util.Constants;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.settings.Setting;
@ -86,10 +88,12 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
final Path location = path.resolveIndex();
Files.createDirectories(location);
Directory wrapped = newFSDirectory(location, indexSettings.getValue(INDEX_LOCK_FACTOR_SETTING));
if (IndexMetaData.isOnSharedFilesystem(indexSettings.getSettings())) {
wrapped = new SleepingLockWrapper(wrapped, 5000);
}
return new RateLimitedFSDirectory(wrapped, this, this) ;
}
@Override
public void onPause(long nanos) {
rateLimitingTimeInNanos.inc(nanos);

4
core/src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -50,7 +50,6 @@ import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
@ -70,7 +69,6 @@ import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
import org.elasticsearch.common.util.concurrent.RefCounted;
import org.elasticsearch.common.util.iterable.Iterables;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
@ -83,7 +81,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.sql.Time;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -150,7 +147,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
this(shardId, indexSettings, directoryService, shardLock, OnClose.EMPTY);
}
@Inject
public Store(ShardId shardId, IndexSettings indexSettings, DirectoryService directoryService, ShardLock shardLock, OnClose onClose) throws IOException {
super(shardId, indexSettings);
final Settings settings = indexSettings.getSettings();

2
core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
@ -79,7 +78,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
/**
* Tests for indices that use shadow replicas and a shared filesystem
*/
@AwaitsFix(bugUrl = "breaks after https://github.com/elastic/elasticsearch/pull/16930 , Simon to fix")
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
public class IndexWithShadowReplicasIT extends ESIntegTestCase {

76
core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.RateLimitedFSDirectory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.store.SleepingLockWrapper;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
public class FsDirectoryServiceTests extends ESTestCase {
public void testHasSleepWrapperOnSharedFS() throws IOException {
Settings build = randomBoolean() ?
Settings.builder().put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true).build() :
Settings.builder().put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).build();;
IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
IndexStoreConfig config = new IndexStoreConfig(build);
IndexStore store = new IndexStore(settings, config);
Path tempDir = createTempDir().resolve("foo").resolve("0");
Files.createDirectories(tempDir);
ShardPath path = new ShardPath(false, tempDir, tempDir, settings.getUUID(), new ShardId(settings.getIndex(), 0));
FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
Directory directory = fsDirectoryService.newDirectory();
assertTrue(directory instanceof RateLimitedFSDirectory);
RateLimitedFSDirectory rateLimitingDirectory = (RateLimitedFSDirectory) directory;
Directory delegate = rateLimitingDirectory.getDelegate();
assertTrue(delegate.getClass().toString(), delegate instanceof SleepingLockWrapper);
}
public void testHasNoSleepWrapperOnNormalFS() throws IOException {
Settings build = Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "simplefs").build();
IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
IndexStoreConfig config = new IndexStoreConfig(build);
IndexStore store = new IndexStore(settings, config);
Path tempDir = createTempDir().resolve("foo").resolve("0");
Files.createDirectories(tempDir);
ShardPath path = new ShardPath(false, tempDir, tempDir, settings.getUUID(), new ShardId(settings.getIndex(), 0));
FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
Directory directory = fsDirectoryService.newDirectory();
assertTrue(directory instanceof RateLimitedFSDirectory);
RateLimitedFSDirectory rateLimitingDirectory = (RateLimitedFSDirectory) directory;
Directory delegate = rateLimitingDirectory.getDelegate();
assertFalse(delegate instanceof SleepingLockWrapper);
assertTrue(delegate instanceof SimpleFSDirectory);
}
}