honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable searchable snapshots in release builds (#62201) 

Enables searchable snapshot functionality not only in snapshot, but also release builds.
This commit is contained in:
Yannick Welsch 2020-09-10 11:06:29 +02:00
parent 969a1c558b
commit e3feafc1e9
13 changed files with 68 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/build.gradle
View File

@ -54,7 +54,6 @@ testClusters.integTest {
setting 'indices.lifecycle.history_index_enabled', 'false'
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.autoscaling_feature_flag_registered', 'true'
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
setting 'xpack.autoscaling.enabled', 'true'
setting 'xpack.eql.enabled', 'true'

3
x-pack/plugin/build.gradle
View File

@ -172,9 +172,6 @@ testClusters.all {
extraConfigFile nodeKey.name, nodeKey
extraConfigFile nodeCert.name, nodeCert
extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
}

21
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstants.java
View File

@ -5,37 +5,18 @@
*/
package org.elasticsearch.xpack.searchablesnapshots;
import org.elasticsearch.Build;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
public class SearchableSnapshotsConstants {
public static final boolean SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED;
static {
final String property = System.getProperty("es.searchable_snapshots_feature_enabled");
if ("true".equals(property)) {
SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED = true;
} else if ("false".equals(property)) {
SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED = false;
} else if (property == null) {
SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED = Build.CURRENT.isSnapshot();
} else {
throw new IllegalArgumentException(
"expected es.searchable_snapshots_feature_enabled to be unset or [true|false] but was [" + property + "]"
);
}
}
public static final String SNAPSHOT_DIRECTORY_FACTORY_KEY = "snapshot";
public static final String SNAPSHOT_RECOVERY_STATE_FACTORY_KEY = "snapshot_prewarm";
public static boolean isSearchableSnapshotStore(Settings indexSettings) {
return SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED
&& SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings));
return SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings));
}
public static final String CACHE_FETCH_ASYNC_THREAD_POOL_NAME = "searchable_snapshots_cache_fetch_async";

5
x-pack/plugin/data-streams/qa/multi-node/build.gradle
View File

@ -1,5 +1,3 @@
import org.elasticsearch.gradle.info.BuildParams
apply plugin: 'elasticsearch.java-rest-test'
File repoDir = file("$buildDir/testclusters/repo")
@ -11,9 +9,6 @@ javaRestTest {
testClusters.javaRestTest {
testDistribution = 'DEFAULT'
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
numberOfNodes = 4
setting 'path.repo', repoDir.absolutePath

5
x-pack/plugin/data-streams/qa/rest/build.gradle
View File

@ -1,5 +1,3 @@
import org.elasticsearch.gradle.info.BuildParams
apply plugin: 'elasticsearch.yaml-rest-test'
restResources {
@ -14,9 +12,6 @@ restResources {
testClusters.all {
testDistribution = 'DEFAULT'
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
// Data streams is basic, but a few tests test data streams in combination with paid features
setting 'xpack.license.self_generated.type', 'trial'
// disable ILM history, since it disturbs tests using _all

6
x-pack/plugin/ilm/qa/multi-node/build.gradle
View File

@ -1,6 +1,3 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.info.BuildParams
apply plugin: 'elasticsearch.java-rest-test'
dependencies {
@ -16,9 +13,6 @@ javaRestTest {
testClusters.all {
testDistribution = 'DEFAULT'
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
numberOfNodes = 4
setting 'path.repo', repoDir.absolutePath

9
x-pack/plugin/searchable-snapshots/build.gradle
View File

@ -1,5 +1,3 @@
import org.elasticsearch.gradle.info.BuildParams
apply plugin: 'elasticsearch.internal-cluster-test'
apply plugin: 'elasticsearch.esplugin'
esplugin {
@ -39,10 +37,3 @@ def testJar = tasks.register("testJar", Jar) {
artifacts {
testArtifacts testJar
}
internalClusterTest {
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
}

4
x-pack/plugin/searchable-snapshots/qa/azure/build.gradle
View File

@ -51,10 +51,6 @@ testClusters.integTest {
testDistribution = 'DEFAULT'
plugin repositoryPlugin.path
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
keystore 'azure.client.searchable_snapshots.account', azureAccount
if (azureKey != null && azureKey.isEmpty() == false) {
keystore 'azure.client.searchable_snapshots.key', azureKey

4
x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle
View File

@ -95,10 +95,6 @@ testClusters.integTest {
testDistribution = 'DEFAULT'
plugin repositoryPlugin.path
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
keystore 'gcs.client.searchable_snapshots.credentials_file', serviceAccountFile, IGNORE_VALUE
if (useFixture) {
tasks.integTest.dependsOn createServiceAccountFile

5
x-pack/plugin/searchable-snapshots/qa/minio/build.gradle
View File

@ -1,4 +1,3 @@
import org.elasticsearch.gradle.info.BuildParams
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.standalone-rest-test'
@ -37,10 +36,6 @@ testClusters.integTest {
testDistribution = 'DEFAULT'
plugin repositoryPlugin.path
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
keystore 's3.client.searchable_snapshots.access_key', 'access_key'
keystore 's3.client.searchable_snapshots.secret_key', 'secret_key'
setting 'xpack.license.self_generated.type', 'trial'

5
x-pack/plugin/searchable-snapshots/qa/rest/build.gradle
View File

@ -1,5 +1,3 @@
import org.elasticsearch.gradle.info.BuildParams
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
@ -17,9 +15,6 @@ integTest {
testClusters.integTest {
testDistribution = 'DEFAULT'
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
setting 'path.repo', repoDir.absolutePath
setting 'xpack.license.self_generated.type', 'trial'
}

4
x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
View File

@ -51,10 +51,6 @@ testClusters.integTest {
testDistribution = 'DEFAULT'
plugin repositoryPlugin.path
if (BuildParams.isSnapshotBuild() == false) {
systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
}
keystore 's3.client.searchable_snapshots.access_key', s3AccessKey
keystore 's3.client.searchable_snapshots.secret_key', s3SecretKey
setting 'xpack.license.self_generated.type', 'trial'

181
x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
View File

@ -82,7 +82,6 @@ import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsCon
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_FETCH_ASYNC_THREAD_POOL_SETTING;
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_PREWARMING_THREAD_POOL_NAME;
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_PREWARMING_THREAD_POOL_SETTING;
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED;
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_BLOB_CACHE_INDEX;
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY;
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_RECOVERY_STATE_FACTORY_KEY;
@ -173,23 +172,19 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng
@Override
public List<Setting<?>> getSettings() {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return org.elasticsearch.common.collect.List.of(
SNAPSHOT_REPOSITORY_SETTING,
SNAPSHOT_SNAPSHOT_NAME_SETTING,
SNAPSHOT_SNAPSHOT_ID_SETTING,
SNAPSHOT_INDEX_NAME_SETTING,
SNAPSHOT_INDEX_ID_SETTING,
SNAPSHOT_CACHE_ENABLED_SETTING,
SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING,
SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING,
SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING,
CacheService.SNAPSHOT_CACHE_SIZE_SETTING,
CacheService.SNAPSHOT_CACHE_RANGE_SIZE_SETTING
);
} else {
return org.elasticsearch.common.collect.List.of();
}
return org.elasticsearch.common.collect.List.of(
SNAPSHOT_REPOSITORY_SETTING,
SNAPSHOT_SNAPSHOT_NAME_SETTING,
SNAPSHOT_SNAPSHOT_ID_SETTING,
SNAPSHOT_INDEX_NAME_SETTING,
SNAPSHOT_INDEX_ID_SETTING,
SNAPSHOT_CACHE_ENABLED_SETTING,
SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING,
SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING,
SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING,
CacheService.SNAPSHOT_CACHE_SIZE_SETTING,
CacheService.SNAPSHOT_CACHE_RANGE_SIZE_SETTING
);
}
@Override
@ -206,42 +201,28 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng
final IndexNameExpressionResolver resolver,
final Supplier<RepositoriesService> repositoriesServiceSupplier
) {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
final CacheService cacheService = new CacheService(new NodeEnvironmentCacheCleaner(nodeEnvironment), settings);
this.cacheService.set(cacheService);
this.repositoriesServiceSupplier = repositoriesServiceSupplier;
this.threadPool.set(threadPool);
final BlobStoreCacheService blobStoreCacheService = new BlobStoreCacheService(
clusterService,
threadPool,
client,
SNAPSHOT_BLOB_CACHE_INDEX
);
this.blobStoreCacheService.set(blobStoreCacheService);
this.failShardsListener.set(
new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), clusterService.getRerouteService())
);
return org.elasticsearch.common.collect.List.of(cacheService, blobStoreCacheService);
} else {
this.repositoriesServiceSupplier = () -> {
assert false : "searchable snapshots are disabled";
return null;
};
return org.elasticsearch.common.collect.List.of();
}
final CacheService cacheService = new CacheService(new NodeEnvironmentCacheCleaner(nodeEnvironment), settings);
this.cacheService.set(cacheService);
this.repositoriesServiceSupplier = repositoriesServiceSupplier;
this.threadPool.set(threadPool);
final BlobStoreCacheService blobStoreCacheService = new BlobStoreCacheService(
clusterService,
threadPool,
client,
SNAPSHOT_BLOB_CACHE_INDEX
);
this.blobStoreCacheService.set(blobStoreCacheService);
this.failShardsListener.set(new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), clusterService.getRerouteService()));
return org.elasticsearch.common.collect.List.of(cacheService, blobStoreCacheService);
}
@Override
public Collection<Module> createGuiceModules() {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
if (transportClientMode) {
return Collections.emptyList();
}
return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, SearchableSnapshotsFeatureSet.class));
} else {
return org.elasticsearch.common.collect.List.of();
if (transportClientMode) {
return Collections.emptyList();
}
return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, SearchableSnapshotsFeatureSet.class));
}
@Override
@ -261,29 +242,25 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng
@Override
public Map<String, DirectoryFactory> getDirectoryFactories() {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return org.elasticsearch.common.collect.Map.of(SNAPSHOT_DIRECTORY_FACTORY_KEY, (indexSettings, shardPath) -> {
final RepositoriesService repositories = repositoriesServiceSupplier.get();
assert repositories != null;
final CacheService cache = cacheService.get();
assert cache != null;
final ThreadPool threadPool = this.threadPool.get();
assert threadPool != null;
final BlobStoreCacheService blobCache = blobStoreCacheService.get();
assert blobCache != null;
return SearchableSnapshotDirectory.create(
repositories,
cache,
indexSettings,
shardPath,
System::nanoTime,
threadPool,
blobCache
);
});
} else {
return org.elasticsearch.common.collect.Map.of();
}
return org.elasticsearch.common.collect.Map.of(SNAPSHOT_DIRECTORY_FACTORY_KEY, (indexSettings, shardPath) -> {
final RepositoriesService repositories = repositoriesServiceSupplier.get();
assert repositories != null;
final CacheService cache = cacheService.get();
assert cache != null;
final ThreadPool threadPool = this.threadPool.get();
assert threadPool != null;
final BlobStoreCacheService blobCache = blobStoreCacheService.get();
assert blobCache != null;
return SearchableSnapshotDirectory.create(
repositories,
cache,
indexSettings,
shardPath,
System::nanoTime,
threadPool,
blobCache
);
});
}
@Override
@ -299,16 +276,12 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng
@Override
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return org.elasticsearch.common.collect.List.of(
new ActionHandler<>(SearchableSnapshotsStatsAction.INSTANCE, TransportSearchableSnapshotsStatsAction.class),
new ActionHandler<>(ClearSearchableSnapshotsCacheAction.INSTANCE, TransportClearSearchableSnapshotsCacheAction.class),
new ActionHandler<>(MountSearchableSnapshotAction.INSTANCE, TransportMountSearchableSnapshotAction.class),
new ActionHandler<>(RepositoryStatsAction.INSTANCE, TransportRepositoryStatsAction.class)
);
} else {
return org.elasticsearch.common.collect.List.of();
}
return org.elasticsearch.common.collect.List.of(
new ActionHandler<>(SearchableSnapshotsStatsAction.INSTANCE, TransportSearchableSnapshotsStatsAction.class),
new ActionHandler<>(ClearSearchableSnapshotsCacheAction.INSTANCE, TransportClearSearchableSnapshotsCacheAction.class),
new ActionHandler<>(MountSearchableSnapshotAction.INSTANCE, TransportMountSearchableSnapshotAction.class),
new ActionHandler<>(RepositoryStatsAction.INSTANCE, TransportRepositoryStatsAction.class)
);
}
public List<RestHandler> getRestHandlers(
@ -320,25 +293,17 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<DiscoveryNodes> nodesInCluster
) {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return org.elasticsearch.common.collect.List.of(
new RestSearchableSnapshotsStatsAction(),
new RestClearSearchableSnapshotsCacheAction(),
new RestMountSearchableSnapshotAction(),
new RestRepositoryStatsAction()
);
} else {
return org.elasticsearch.common.collect.List.of();
}
return org.elasticsearch.common.collect.List.of(
new RestSearchableSnapshotsStatsAction(),
new RestClearSearchableSnapshotsCacheAction(),
new RestMountSearchableSnapshotAction(),
new RestRepositoryStatsAction()
);
}
@Override
public Map<String, ExistingShardsAllocator> getExistingShardsAllocators() {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return Collections.singletonMap(SearchableSnapshotAllocator.ALLOCATOR_NAME, new SearchableSnapshotAllocator());
} else {
return Collections.emptyMap();
}
return org.elasticsearch.common.collect.Map.of(SearchableSnapshotAllocator.ALLOCATOR_NAME, new SearchableSnapshotAllocator());
}
// overridable by tests
@ -348,30 +313,18 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng
@Override
public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return org.elasticsearch.common.collect.List.of(
new SearchableSnapshotAllocationDecider(() -> getLicenseState().isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS))
);
} else {
return Collections.emptyList();
}
return org.elasticsearch.common.collect.List.of(
new SearchableSnapshotAllocationDecider(() -> getLicenseState().isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS))
);
}
public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return org.elasticsearch.common.collect.List.of(executorBuilders());
} else {
return org.elasticsearch.common.collect.List.of();
}
return org.elasticsearch.common.collect.List.of(executorBuilders());
}
@Override
public Map<String, RecoveryStateFactory> getRecoveryStateFactories() {
if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) {
return Collections.singletonMap(SNAPSHOT_RECOVERY_STATE_FACTORY_KEY, SearchableSnapshotRecoveryState::new);
} else {
return Collections.emptyMap();
}
return Collections.singletonMap(SNAPSHOT_RECOVERY_STATE_FACTORY_KEY, SearchableSnapshotRecoveryState::new);
}
public static ScalingExecutorBuilder[] executorBuilders() {