Snapshot/Restore: add support for changing index settings during restore process
Closes #7887
This commit is contained in:
Igor Motov
parent
81621840b8
commit
c0da353ef5
|
|
@ -243,6 +243,24 @@ restore such indices by setting `partial` to `true`. Please note, that only succ
|
|||
restored in this case and all missing shards will be recreated empty.
|
||||
|
||||
|
||||
[float]
|
||||
=== Changing index settings during restore
|
||||
|
||||
Most of index settings can be overridden during the restore process. For example, the following command will restore
|
||||
the index `index_1` without creating any replicas while switching back to default refresh interval:
|
||||
|
||||
[source,js]
|
||||
-----------------------------------
|
||||
$ curl -XPOST "localhost:9200/_snapshot/my_backup/snapshot_1/_restore" -d '{
|
||||
"indices": "index_1",
|
||||
"index_settings" : {
|
||||
"index.number_of_replicas": 0
|
||||
},
|
||||
"ignore_index_settings": ["index.refresh_interval"]
|
||||
}'
|
||||
-----------------------------------
|
||||
|
||||
|
||||
[float]
|
||||
=== Snapshot status
|
||||
|
||||
|
|
|
|||
|
|
@ -74,6 +74,10 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
|
||||
private Settings settings = EMPTY_SETTINGS;
|
||||
|
||||
private Settings indexSettings = EMPTY_SETTINGS;
|
||||
|
||||
private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
|
||||
|
||||
RestoreSnapshotRequest() {
|
||||
}
|
||||
|
||||
|
|
@ -106,7 +110,12 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
if (settings == null) {
|
||||
validationException = addValidationError("settings are missing", validationException);
|
||||
}
|
||||
|
||||
if (indexSettings == null) {
|
||||
validationException = addValidationError("indexSettings are missing", validationException);
|
||||
}
|
||||
if (ignoreIndexSettings == null) {
|
||||
validationException = addValidationError("ignoreIndexSettings are missing", validationException);
|
||||
}
|
||||
return validationException;
|
||||
}
|
||||
|
||||
|
|
@ -364,6 +373,29 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
return this.settings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
|
||||
*/
|
||||
public RestoreSnapshotRequest ignoreIndexSettings(String... ignoreIndexSettings) {
|
||||
this.ignoreIndexSettings = ignoreIndexSettings;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
|
||||
*/
|
||||
public RestoreSnapshotRequest ignoreIndexSettings(List<String> ignoreIndexSettings) {
|
||||
this.ignoreIndexSettings = ignoreIndexSettings.toArray(new String[ignoreIndexSettings.size()]);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the list of index settings and index settings groups that shouldn't be restored from snapshot
|
||||
*/
|
||||
public String[] ignoreIndexSettings() {
|
||||
return ignoreIndexSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* If set to true the restore procedure will restore global cluster state.
|
||||
* <p/>
|
||||
|
|
@ -406,6 +438,51 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
return includeAliases;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets settings that should be added/changed in all restored indices
|
||||
*/
|
||||
public RestoreSnapshotRequest indexSettings(Settings settings) {
|
||||
this.indexSettings = settings;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets settings that should be added/changed in all restored indices
|
||||
*/
|
||||
public RestoreSnapshotRequest indexSettings(Settings.Builder settings) {
|
||||
this.indexSettings = settings.build();
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets settings that should be added/changed in all restored indices
|
||||
*/
|
||||
public RestoreSnapshotRequest indexSettings(String source) {
|
||||
this.indexSettings = ImmutableSettings.settingsBuilder().loadFromSource(source).build();
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets settings that should be added/changed in all restored indices
|
||||
*/
|
||||
public RestoreSnapshotRequest indexSettings(Map<String, Object> source) {
|
||||
try {
|
||||
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
|
||||
builder.map(source);
|
||||
indexSettings(builder.string());
|
||||
} catch (IOException e) {
|
||||
throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns settings that should be added/changed in all restored indices
|
||||
*/
|
||||
public Settings indexSettings() {
|
||||
return this.indexSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses restore definition
|
||||
*
|
||||
|
|
@ -454,7 +531,7 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
partial(nodeBooleanValue(entry.getValue()));
|
||||
} else if (name.equals("settings")) {
|
||||
if (!(entry.getValue() instanceof Map)) {
|
||||
throw new ElasticsearchIllegalArgumentException("malformed settings section, should indices an inner object");
|
||||
throw new ElasticsearchIllegalArgumentException("malformed settings section");
|
||||
}
|
||||
settings((Map<String, Object>) entry.getValue());
|
||||
} else if (name.equals("include_global_state")) {
|
||||
|
|
@ -473,6 +550,19 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
} else {
|
||||
throw new ElasticsearchIllegalArgumentException("malformed rename_replacement");
|
||||
}
|
||||
} else if (name.equals("index_settings")) {
|
||||
if (!(entry.getValue() instanceof Map)) {
|
||||
throw new ElasticsearchIllegalArgumentException("malformed index_settings section");
|
||||
}
|
||||
indexSettings((Map<String, Object>) entry.getValue());
|
||||
} else if (name.equals("ignore_index_settings")) {
|
||||
if (entry.getValue() instanceof String) {
|
||||
ignoreIndexSettings(Strings.splitStringByCommaToArray((String) entry.getValue()));
|
||||
} else if (entry.getValue() instanceof List) {
|
||||
ignoreIndexSettings((List<String>) entry.getValue());
|
||||
} else {
|
||||
throw new ElasticsearchIllegalArgumentException("malformed ignore_index_settings section, should be an array of strings");
|
||||
}
|
||||
} else {
|
||||
throw new ElasticsearchIllegalArgumentException("Unknown parameter " + name);
|
||||
}
|
||||
|
|
@ -563,6 +653,10 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
partial = in.readBoolean();
|
||||
includeAliases = in.readBoolean();
|
||||
settings = readSettingsFromStream(in);
|
||||
if (in.getVersion().onOrAfter(Version.V_1_5_0)) {
|
||||
indexSettings = readSettingsFromStream(in);
|
||||
ignoreIndexSettings = in.readStringArray();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -579,5 +673,9 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
|
|||
out.writeBoolean(partial);
|
||||
out.writeBoolean(includeAliases);
|
||||
writeSettingsToStream(settings, out);
|
||||
if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
|
||||
writeSettingsToStream(indexSettings, out);
|
||||
out.writeStringArray(ignoreIndexSettings);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder
|
|||
import org.elasticsearch.client.ClusterAdminClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
|
|
@ -230,6 +231,68 @@ public class RestoreSnapshotRequestBuilder extends MasterNodeOperationRequestBui
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets index settings that should be added or replaced during restore
|
||||
|
||||
* @param settings index settings
|
||||
* @return this builder
|
||||
*/
|
||||
public RestoreSnapshotRequestBuilder setIndexSettings(Settings settings) {
|
||||
request.indexSettings(settings);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets index settings that should be added or replaced during restore
|
||||
|
||||
* @param settings index settings
|
||||
* @return this builder
|
||||
*/
|
||||
public RestoreSnapshotRequestBuilder setIndexSettings(Settings.Builder settings) {
|
||||
request.indexSettings(settings);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets index settings that should be added or replaced during restore
|
||||
|
||||
* @param source index settings
|
||||
* @return this builder
|
||||
*/
|
||||
public RestoreSnapshotRequestBuilder setIndexSettings(String source) {
|
||||
request.indexSettings(source);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets index settings that should be added or replaced during restore
|
||||
|
||||
* @param source index settings
|
||||
* @return this builder
|
||||
*/
|
||||
public RestoreSnapshotRequestBuilder setIndexSettings(Map<String, Object> source) {
|
||||
request.indexSettings(source);
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
|
||||
*/
|
||||
public RestoreSnapshotRequestBuilder setIgnoreIndexSettings(String... ignoreIndexSettings) {
|
||||
request.ignoreIndexSettings(ignoreIndexSettings);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
|
||||
*/
|
||||
public RestoreSnapshotRequestBuilder setIgnoreIndexSettings(List<String> ignoreIndexSettings) {
|
||||
request.ignoreIndexSettings(ignoreIndexSettings);
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void doExecute(ActionListener<RestoreSnapshotResponse> listener) {
|
||||
client.restoreSnapshot(request, listener);
|
||||
|
|
|
|||
|
|
@ -73,7 +73,8 @@ public class TransportRestoreSnapshotAction extends TransportMasterNodeOperation
|
|||
RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest(
|
||||
"restore_snapshot[" + request.snapshot() + "]", request.repository(), request.snapshot(),
|
||||
request.indices(), request.indicesOptions(), request.renamePattern(), request.renameReplacement(),
|
||||
request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases());
|
||||
request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases(),
|
||||
request.indexSettings(), request.ignoreIndexSettings());
|
||||
|
||||
restoreService.restoreSnapshot(restoreRequest, new ActionListener<RestoreInfo>() {
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
|
|||
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.support.IndicesOptions;
|
||||
|
|
@ -41,6 +42,7 @@ import org.elasticsearch.common.component.AbstractComponent;
|
|||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
|
|
@ -57,6 +59,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
|
|||
import static com.google.common.collect.Lists.newArrayList;
|
||||
import static com.google.common.collect.Maps.newHashMap;
|
||||
import static com.google.common.collect.Sets.newHashSet;
|
||||
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
|
||||
import static org.elasticsearch.cluster.metadata.MetaDataIndexStateService.INDEX_CLOSED_BLOCK;
|
||||
|
||||
/**
|
||||
|
|
@ -85,6 +88,21 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
|
|||
|
||||
public static final String UPDATE_RESTORE_ACTION_NAME = "internal:cluster/snapshot/update_restore";
|
||||
|
||||
private static final ImmutableSet<String> UNMODIFIABLE_SETTINGS = ImmutableSet.of(
|
||||
SETTING_NUMBER_OF_SHARDS,
|
||||
SETTING_VERSION_CREATED,
|
||||
SETTING_LEGACY_ROUTING_HASH_FUNCTION,
|
||||
SETTING_LEGACY_ROUTING_USE_TYPE,
|
||||
SETTING_UUID,
|
||||
SETTING_CREATION_DATE);
|
||||
|
||||
// It's OK to change some settings, but we shouldn't allow simply removing them
|
||||
private static final ImmutableSet<String> UNREMOVABLE_SETTINGS = ImmutableSet.<String>builder()
|
||||
.addAll(UNMODIFIABLE_SETTINGS)
|
||||
.add(SETTING_NUMBER_OF_REPLICAS)
|
||||
.add(SETTING_AUTO_EXPAND_REPLICAS)
|
||||
.build();
|
||||
|
||||
private final ClusterService clusterService;
|
||||
|
||||
private final RepositoriesService repositoriesService;
|
||||
|
|
@ -163,6 +181,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
|
|||
RestoreSource restoreSource = new RestoreSource(snapshotId, index);
|
||||
String renamedIndex = indexEntry.getKey();
|
||||
IndexMetaData snapshotIndexMetaData = metaData.index(index);
|
||||
snapshotIndexMetaData = updateIndexSettings(snapshotIndexMetaData, request.indexSettings, request.ignoreIndexSettings);
|
||||
// Check that the index is closed or doesn't exist
|
||||
IndexMetaData currentIndexMetaData = currentState.metaData().index(renamedIndex);
|
||||
IntSet ignoreShards = new IntOpenHashSet();
|
||||
|
|
@ -287,6 +306,51 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Optionally updates index settings in indexMetaData by removing settings listed in ignoreSettings and
|
||||
* merging them with settings in changeSettings.
|
||||
*/
|
||||
private IndexMetaData updateIndexSettings(IndexMetaData indexMetaData, Settings changeSettings, String[] ignoreSettings) {
|
||||
if (changeSettings.names().isEmpty() && ignoreSettings.length == 0) {
|
||||
return indexMetaData;
|
||||
}
|
||||
IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData);
|
||||
Map<String, String> settingsMap = newHashMap(indexMetaData.settings().getAsMap());
|
||||
List<String> simpleMatchPatterns = newArrayList();
|
||||
for (String ignoredSetting : ignoreSettings) {
|
||||
if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
|
||||
if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
|
||||
throw new SnapshotRestoreException(snapshotId, "cannot remove setting [" + ignoredSetting + "] on restore");
|
||||
} else {
|
||||
settingsMap.remove(ignoredSetting);
|
||||
}
|
||||
} else {
|
||||
simpleMatchPatterns.add(ignoredSetting);
|
||||
}
|
||||
}
|
||||
if (!simpleMatchPatterns.isEmpty()) {
|
||||
String[] removePatterns = simpleMatchPatterns.toArray(new String[simpleMatchPatterns.size()]);
|
||||
Iterator<Map.Entry<String, String>> iterator = settingsMap.entrySet().iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Map.Entry<String, String> entry = iterator.next();
|
||||
if (UNREMOVABLE_SETTINGS.contains(entry.getKey()) == false) {
|
||||
if (Regex.simpleMatch(removePatterns, entry.getKey())) {
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for(Map.Entry<String, String> entry : changeSettings.getAsMap().entrySet()) {
|
||||
if (UNMODIFIABLE_SETTINGS.contains(entry.getKey())) {
|
||||
throw new SnapshotRestoreException(snapshotId, "cannot modify setting [" + entry.getKey() + "] on restore");
|
||||
} else {
|
||||
settingsMap.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
return builder.settings(ImmutableSettings.builder().put(settingsMap)).build();
|
||||
}
|
||||
|
||||
private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) {
|
||||
if (request.includeGlobalState()) {
|
||||
if (metaData.persistentSettings() != null) {
|
||||
|
|
@ -703,6 +767,10 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
|
|||
|
||||
final private boolean includeAliases;
|
||||
|
||||
final private Settings indexSettings;
|
||||
|
||||
final private String[] ignoreIndexSettings;
|
||||
|
||||
/**
|
||||
* Constructs new restore request
|
||||
*
|
||||
|
|
@ -717,10 +785,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
|
|||
* @param masterNodeTimeout master node timeout
|
||||
* @param includeGlobalState include global state into restore
|
||||
* @param partial allow partial restore
|
||||
* @param indexSettings index settings that should be changed on restore
|
||||
* @param ignoreIndexSettings index settings that shouldn't be restored
|
||||
*/
|
||||
public RestoreRequest(String cause, String repository, String name, String[] indices, IndicesOptions indicesOptions,
|
||||
String renamePattern, String renameReplacement, Settings settings,
|
||||
TimeValue masterNodeTimeout, boolean includeGlobalState, boolean partial, boolean includeAliases) {
|
||||
TimeValue masterNodeTimeout, boolean includeGlobalState, boolean partial, boolean includeAliases,
|
||||
Settings indexSettings, String[] ignoreIndexSettings ) {
|
||||
this.cause = cause;
|
||||
this.name = name;
|
||||
this.repository = repository;
|
||||
|
|
@ -733,6 +804,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
|
|||
this.includeGlobalState = includeGlobalState;
|
||||
this.partial = partial;
|
||||
this.includeAliases = includeAliases;
|
||||
this.indexSettings = indexSettings;
|
||||
this.ignoreIndexSettings = ignoreIndexSettings;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -834,6 +908,25 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
|
|||
return includeAliases;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns index settings that should be changed on restore
|
||||
*
|
||||
* @return restore aliases state flag
|
||||
*/
|
||||
public Settings indexSettings() {
|
||||
return indexSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns index settings that that shouldn't be restored
|
||||
*
|
||||
* @return restore aliases state flag
|
||||
*/
|
||||
public String[] ignoreIndexSettings() {
|
||||
return ignoreIndexSettings;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return master node timeout
|
||||
*
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import org.elasticsearch.cluster.metadata.SnapshotMetaData;
|
|||
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
|
||||
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.index.store.support.AbstractIndexStore;
|
||||
import org.elasticsearch.indices.InvalidIndexNameException;
|
||||
|
|
@ -64,7 +65,9 @@ import java.util.concurrent.TimeUnit;
|
|||
|
||||
import static com.google.common.collect.Lists.newArrayList;
|
||||
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
|
||||
import static org.elasticsearch.index.shard.IndexShard.*;
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
@Slow
|
||||
|
|
@ -77,7 +80,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))
|
||||
.put("location", newTempDirPath())
|
||||
.put("compress", randomBoolean())
|
||||
.put("chunk_size", randomIntBetween(100, 1000))));
|
||||
|
||||
|
|
@ -216,7 +219,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))
|
||||
.put("location", newTempDirPath())
|
||||
.put("compress", randomBoolean())
|
||||
.put("chunk_size", randomIntBetween(100, 1000))));
|
||||
|
||||
|
|
@ -451,7 +454,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
|
||||
ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.TEST))
|
||||
.put("location", newTempDirPath())
|
||||
.put("random", randomAsciiOfLength(10))
|
||||
.put("random_control_io_exception_rate", 0.2))
|
||||
.setVerify(false));
|
||||
|
|
@ -501,7 +504,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
|
||||
ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.TEST))
|
||||
.put("location", newTempDirPath())
|
||||
.put("random", randomAsciiOfLength(10))
|
||||
.put("random_data_file_io_exception_rate", 0.3)));
|
||||
|
||||
|
|
@ -563,7 +566,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
|
||||
@Test
|
||||
public void dataFileFailureDuringRestoreTest() throws Exception {
|
||||
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST);
|
||||
Path repositoryLocation = newTempDirPath();
|
||||
Client client = client();
|
||||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
|
|
@ -605,7 +608,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
|
||||
@Test
|
||||
public void deletionOfFailingToRecoverIndexShouldStopRestore() throws Exception {
|
||||
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST);
|
||||
Path repositoryLocation = newTempDirPath();
|
||||
Client client = client();
|
||||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
|
|
@ -674,7 +677,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))));
|
||||
.put("location", newTempDirPath())));
|
||||
|
||||
logger.info("--> creating index that cannot be allocated");
|
||||
prepareCreate("test-idx", 2, ImmutableSettings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + ".tag", "nowhere").put("index.number_of_shards", 3)).get();
|
||||
|
|
@ -692,7 +695,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
final int numberOfSnapshots = between(5, 15);
|
||||
Client client = client();
|
||||
|
||||
Path repo = newTempDirPath(LifecycleScope.SUITE);
|
||||
Path repo = newTempDirPath();
|
||||
logger.info("--> creating repository at " + repo.toAbsolutePath());
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
|
|
@ -749,7 +752,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
public void deleteSnapshotWithMissingIndexAndShardMetadataTest() throws Exception {
|
||||
Client client = client();
|
||||
|
||||
Path repo = newTempDirPath(LifecycleScope.SUITE);
|
||||
Path repo = newTempDirPath();
|
||||
logger.info("--> creating repository at " + repo.toAbsolutePath());
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
|
|
@ -788,7 +791,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
public void deleteSnapshotWithMissingMetadataTest() throws Exception {
|
||||
Client client = client();
|
||||
|
||||
Path repo = newTempDirPath(LifecycleScope.SUITE);
|
||||
Path repo = newTempDirPath();
|
||||
logger.info("--> creating repository at " + repo.toAbsolutePath());
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
|
|
@ -826,7 +829,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))));
|
||||
.put("location", newTempDirPath())));
|
||||
|
||||
createIndex("test-idx", "test-idx-closed");
|
||||
ensureGreen();
|
||||
|
|
@ -852,7 +855,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))));
|
||||
.put("location", newTempDirPath())));
|
||||
|
||||
createIndex("test-idx");
|
||||
ensureGreen();
|
||||
|
|
@ -873,7 +876,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))));
|
||||
.put("location", newTempDirPath())));
|
||||
|
||||
createIndex("test-idx-1", "test-idx-2", "test-idx-3");
|
||||
ensureGreen();
|
||||
|
|
@ -989,7 +992,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
@Test
|
||||
public void moveShardWhileSnapshottingTest() throws Exception {
|
||||
Client client = client();
|
||||
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST);
|
||||
Path repositoryLocation = newTempDirPath();
|
||||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
|
||||
|
|
@ -1051,7 +1054,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
@Test
|
||||
public void deleteRepositoryWhileSnapshottingTest() throws Exception {
|
||||
Client client = client();
|
||||
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST);
|
||||
Path repositoryLocation = newTempDirPath();
|
||||
logger.info("--> creating repository");
|
||||
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
|
||||
|
|
@ -1136,7 +1139,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
Client client = client();
|
||||
|
||||
logger.info("--> creating repository");
|
||||
Path repositoryLocation = newTempDirPath(LifecycleScope.SUITE);
|
||||
Path repositoryLocation = newTempDirPath();
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", repositoryLocation)
|
||||
|
|
@ -1194,7 +1197,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
Client client = client();
|
||||
|
||||
logger.info("--> creating repository");
|
||||
Path repositoryLocation = newTempDirPath(LifecycleScope.SUITE);
|
||||
Path repositoryLocation = newTempDirPath();
|
||||
boolean throttleSnapshot = randomBoolean();
|
||||
boolean throttleRestore = randomBoolean();
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
|
|
@ -1252,7 +1255,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
@Test
|
||||
public void snapshotStatusTest() throws Exception {
|
||||
Client client = client();
|
||||
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST);
|
||||
Path repositoryLocation = newTempDirPath();
|
||||
logger.info("--> creating repository");
|
||||
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
|
||||
|
|
@ -1347,7 +1350,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))
|
||||
.put("location", newTempDirPath())
|
||||
.put("compress", randomBoolean())
|
||||
.put("chunk_size", randomIntBetween(100, 1000))));
|
||||
|
||||
|
|
@ -1395,7 +1398,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath(LifecycleScope.SUITE))
|
||||
.put("location", newTempDirPath())
|
||||
.put("compress", randomBoolean())
|
||||
.put("chunk_size", randomIntBetween(100, 1000))));
|
||||
|
||||
|
|
@ -1451,6 +1454,110 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void changeSettingsOnRestoreTest() throws Exception {
|
||||
Client client = client();
|
||||
|
||||
logger.info("--> creating repository");
|
||||
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
|
||||
.put("location", newTempDirPath())
|
||||
.put("compress", randomBoolean())
|
||||
.put("chunk_size", randomIntBetween(100, 1000))));
|
||||
|
||||
logger.info("--> create test index with synonyms search analyzer");
|
||||
|
||||
ImmutableSettings.Builder indexSettings = ImmutableSettings.builder()
|
||||
.put(indexSettings())
|
||||
.put(SETTING_NUMBER_OF_REPLICAS, between(0, 1))
|
||||
.put(INDEX_REFRESH_INTERVAL, "10s")
|
||||
.put("index.analysis.analyzer.my_analyzer.type", "custom")
|
||||
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
|
||||
.putArray("index.analysis.analyzer.my_analyzer.filter", "lowercase", "my_synonym")
|
||||
.put("index.analysis.filter.my_synonym.type", "synonym")
|
||||
.put("index.analysis.filter.my_synonym.synonyms", "foo => bar");
|
||||
|
||||
assertAcked(prepareCreate("test-idx", 2, indexSettings));
|
||||
|
||||
int numberOfShards = getNumShards("test-idx").numPrimaries;
|
||||
assertAcked(client().admin().indices().preparePutMapping("test-idx").setType("type1").setSource("field1", "type=string,search_analyzer=my_analyzer"));
|
||||
final int numdocs = randomIntBetween(10, 100);
|
||||
IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
|
||||
for (int i = 0; i < builders.length; i++) {
|
||||
builders[i] = client().prepareIndex("test-idx", "type1", Integer.toString(i)).setSource("field1", "bar " + i);
|
||||
}
|
||||
indexRandom(true, builders);
|
||||
flushAndRefresh();
|
||||
|
||||
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), numdocs);
|
||||
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs);
|
||||
|
||||
logger.info("--> snapshot it");
|
||||
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
|
||||
|
||||
logger.info("--> delete the index and recreate it while changing refresh interval and analyzer");
|
||||
cluster().wipeIndices("test-idx");
|
||||
|
||||
Settings newIndexSettings = ImmutableSettings.builder()
|
||||
.put(INDEX_REFRESH_INTERVAL, "5s")
|
||||
.put("index.analysis.analyzer.my_analyzer.type", "standard")
|
||||
.build();
|
||||
|
||||
Settings newIncorrectIndexSettings = ImmutableSettings.builder()
|
||||
.put(newIndexSettings)
|
||||
.put(SETTING_NUMBER_OF_SHARDS, numberOfShards + 100)
|
||||
.build();
|
||||
|
||||
logger.info("--> try restoring while changing the number of shards - should fail");
|
||||
assertThrows(client.admin().cluster()
|
||||
.prepareRestoreSnapshot("test-repo", "test-snap")
|
||||
.setIgnoreIndexSettings("index.analysis.*")
|
||||
.setIndexSettings(newIncorrectIndexSettings)
|
||||
.setWaitForCompletion(true), SnapshotRestoreException.class);
|
||||
|
||||
logger.info("--> restore index with correct settings from the snapshot");
|
||||
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster()
|
||||
.prepareRestoreSnapshot("test-repo", "test-snap")
|
||||
.setIgnoreIndexSettings("index.analysis.*")
|
||||
.setIndexSettings(newIndexSettings)
|
||||
.setWaitForCompletion(true).execute().actionGet();
|
||||
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
|
||||
logger.info("--> assert that correct settings are restored");
|
||||
GetSettingsResponse getSettingsResponse = client.admin().indices().prepareGetSettings("test-idx").execute().actionGet();
|
||||
assertThat(getSettingsResponse.getSetting("test-idx", INDEX_REFRESH_INTERVAL), equalTo("5s"));
|
||||
// Make sure that number of shards didn't change
|
||||
assertThat(getSettingsResponse.getSetting("test-idx", SETTING_NUMBER_OF_SHARDS), equalTo("" + numberOfShards));
|
||||
assertThat(getSettingsResponse.getSetting("test-idx", "index.analysis.analyzer.my_analyzer.type"), equalTo("standard"));
|
||||
assertThat(getSettingsResponse.getSetting("test-idx", "index.analysis.filter.my_synonym.type"), nullValue());
|
||||
|
||||
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), 0);
|
||||
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs);
|
||||
|
||||
logger.info("--> delete the index and recreate it while deleting all index settings");
|
||||
cluster().wipeIndices("test-idx");
|
||||
|
||||
logger.info("--> restore index with correct settings from the snapshot");
|
||||
restoreSnapshotResponse = client.admin().cluster()
|
||||
.prepareRestoreSnapshot("test-repo", "test-snap")
|
||||
.setIgnoreIndexSettings("*") // delete everything we can delete
|
||||
.setIndexSettings(newIndexSettings)
|
||||
.setWaitForCompletion(true).execute().actionGet();
|
||||
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
|
||||
logger.info("--> assert that correct settings are restored and index is still functional");
|
||||
getSettingsResponse = client.admin().indices().prepareGetSettings("test-idx").execute().actionGet();
|
||||
assertThat(getSettingsResponse.getSetting("test-idx", INDEX_REFRESH_INTERVAL), equalTo("5s"));
|
||||
// Make sure that number of shards didn't change
|
||||
assertThat(getSettingsResponse.getSetting("test-idx", SETTING_NUMBER_OF_SHARDS), equalTo("" + numberOfShards));
|
||||
|
||||
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), 0);
|
||||
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs);
|
||||
|
||||
}
|
||||
|
||||
private boolean waitForIndex(final String index, TimeValue timeout) throws InterruptedException {
|
||||
return awaitBusy(new Predicate<Object>() {
|
||||
@Override
|
||||
|
|
|
|||
Loading…
Reference in New Issue