Snapshot/Restore: add support for changing index settings during restore process

Closes #7887
This commit is contained in:
Igor Motov 2015-01-13 19:16:37 -05:00
parent 81621840b8
commit c0da353ef5
6 changed files with 404 additions and 24 deletions

View File

@ -243,6 +243,24 @@ restore such indices by setting `partial` to `true`. Please note, that only succ
restored in this case and all missing shards will be recreated empty. restored in this case and all missing shards will be recreated empty.
[float]
=== Changing index settings during restore
Most of index settings can be overridden during the restore process. For example, the following command will restore
the index `index_1` without creating any replicas while switching back to default refresh interval:
[source,js]
-----------------------------------
$ curl -XPOST "localhost:9200/_snapshot/my_backup/snapshot_1/_restore" -d '{
"indices": "index_1",
"index_settings" : {
"index.number_of_replicas": 0
},
"ignore_index_settings": ["index.refresh_interval"]
}'
-----------------------------------
[float] [float]
=== Snapshot status === Snapshot status

View File

@ -74,6 +74,10 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
private Settings settings = EMPTY_SETTINGS; private Settings settings = EMPTY_SETTINGS;
private Settings indexSettings = EMPTY_SETTINGS;
private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
RestoreSnapshotRequest() { RestoreSnapshotRequest() {
} }
@ -106,7 +110,12 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
if (settings == null) { if (settings == null) {
validationException = addValidationError("settings are missing", validationException); validationException = addValidationError("settings are missing", validationException);
} }
if (indexSettings == null) {
validationException = addValidationError("indexSettings are missing", validationException);
}
if (ignoreIndexSettings == null) {
validationException = addValidationError("ignoreIndexSettings are missing", validationException);
}
return validationException; return validationException;
} }
@ -364,6 +373,29 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
return this.settings; return this.settings;
} }
/**
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
*/
public RestoreSnapshotRequest ignoreIndexSettings(String... ignoreIndexSettings) {
this.ignoreIndexSettings = ignoreIndexSettings;
return this;
}
/**
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
*/
public RestoreSnapshotRequest ignoreIndexSettings(List<String> ignoreIndexSettings) {
this.ignoreIndexSettings = ignoreIndexSettings.toArray(new String[ignoreIndexSettings.size()]);
return this;
}
/**
* Returns the list of index settings and index settings groups that shouldn't be restored from snapshot
*/
public String[] ignoreIndexSettings() {
return ignoreIndexSettings;
}
/** /**
* If set to true the restore procedure will restore global cluster state. * If set to true the restore procedure will restore global cluster state.
* <p/> * <p/>
@ -406,6 +438,51 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
return includeAliases; return includeAliases;
} }
/**
* Sets settings that should be added/changed in all restored indices
*/
public RestoreSnapshotRequest indexSettings(Settings settings) {
this.indexSettings = settings;
return this;
}
/**
* Sets settings that should be added/changed in all restored indices
*/
public RestoreSnapshotRequest indexSettings(Settings.Builder settings) {
this.indexSettings = settings.build();
return this;
}
/**
* Sets settings that should be added/changed in all restored indices
*/
public RestoreSnapshotRequest indexSettings(String source) {
this.indexSettings = ImmutableSettings.settingsBuilder().loadFromSource(source).build();
return this;
}
/**
* Sets settings that should be added/changed in all restored indices
*/
public RestoreSnapshotRequest indexSettings(Map<String, Object> source) {
try {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
builder.map(source);
indexSettings(builder.string());
} catch (IOException e) {
throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
}
return this;
}
/**
* Returns settings that should be added/changed in all restored indices
*/
public Settings indexSettings() {
return this.indexSettings;
}
/** /**
* Parses restore definition * Parses restore definition
* *
@ -454,7 +531,7 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
partial(nodeBooleanValue(entry.getValue())); partial(nodeBooleanValue(entry.getValue()));
} else if (name.equals("settings")) { } else if (name.equals("settings")) {
if (!(entry.getValue() instanceof Map)) { if (!(entry.getValue() instanceof Map)) {
throw new ElasticsearchIllegalArgumentException("malformed settings section, should indices an inner object"); throw new ElasticsearchIllegalArgumentException("malformed settings section");
} }
settings((Map<String, Object>) entry.getValue()); settings((Map<String, Object>) entry.getValue());
} else if (name.equals("include_global_state")) { } else if (name.equals("include_global_state")) {
@ -473,6 +550,19 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
} else { } else {
throw new ElasticsearchIllegalArgumentException("malformed rename_replacement"); throw new ElasticsearchIllegalArgumentException("malformed rename_replacement");
} }
} else if (name.equals("index_settings")) {
if (!(entry.getValue() instanceof Map)) {
throw new ElasticsearchIllegalArgumentException("malformed index_settings section");
}
indexSettings((Map<String, Object>) entry.getValue());
} else if (name.equals("ignore_index_settings")) {
if (entry.getValue() instanceof String) {
ignoreIndexSettings(Strings.splitStringByCommaToArray((String) entry.getValue()));
} else if (entry.getValue() instanceof List) {
ignoreIndexSettings((List<String>) entry.getValue());
} else {
throw new ElasticsearchIllegalArgumentException("malformed ignore_index_settings section, should be an array of strings");
}
} else { } else {
throw new ElasticsearchIllegalArgumentException("Unknown parameter " + name); throw new ElasticsearchIllegalArgumentException("Unknown parameter " + name);
} }
@ -563,6 +653,10 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
partial = in.readBoolean(); partial = in.readBoolean();
includeAliases = in.readBoolean(); includeAliases = in.readBoolean();
settings = readSettingsFromStream(in); settings = readSettingsFromStream(in);
if (in.getVersion().onOrAfter(Version.V_1_5_0)) {
indexSettings = readSettingsFromStream(in);
ignoreIndexSettings = in.readStringArray();
}
} }
@Override @Override
@ -579,5 +673,9 @@ public class RestoreSnapshotRequest extends MasterNodeOperationRequest<RestoreSn
out.writeBoolean(partial); out.writeBoolean(partial);
out.writeBoolean(includeAliases); out.writeBoolean(includeAliases);
writeSettingsToStream(settings, out); writeSettingsToStream(settings, out);
if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
writeSettingsToStream(indexSettings, out);
out.writeStringArray(ignoreIndexSettings);
}
} }
} }

View File

@ -25,6 +25,7 @@ import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder
import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.client.ClusterAdminClient;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import java.util.List;
import java.util.Map; import java.util.Map;
/** /**
@ -230,6 +231,68 @@ public class RestoreSnapshotRequestBuilder extends MasterNodeOperationRequestBui
return this; return this;
} }
/**
* Sets index settings that should be added or replaced during restore
* @param settings index settings
* @return this builder
*/
public RestoreSnapshotRequestBuilder setIndexSettings(Settings settings) {
request.indexSettings(settings);
return this;
}
/**
* Sets index settings that should be added or replaced during restore
* @param settings index settings
* @return this builder
*/
public RestoreSnapshotRequestBuilder setIndexSettings(Settings.Builder settings) {
request.indexSettings(settings);
return this;
}
/**
* Sets index settings that should be added or replaced during restore
* @param source index settings
* @return this builder
*/
public RestoreSnapshotRequestBuilder setIndexSettings(String source) {
request.indexSettings(source);
return this;
}
/**
* Sets index settings that should be added or replaced during restore
* @param source index settings
* @return this builder
*/
public RestoreSnapshotRequestBuilder setIndexSettings(Map<String, Object> source) {
request.indexSettings(source);
return this;
}
/**
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
*/
public RestoreSnapshotRequestBuilder setIgnoreIndexSettings(String... ignoreIndexSettings) {
request.ignoreIndexSettings(ignoreIndexSettings);
return this;
}
/**
* Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
*/
public RestoreSnapshotRequestBuilder setIgnoreIndexSettings(List<String> ignoreIndexSettings) {
request.ignoreIndexSettings(ignoreIndexSettings);
return this;
}
@Override @Override
protected void doExecute(ActionListener<RestoreSnapshotResponse> listener) { protected void doExecute(ActionListener<RestoreSnapshotResponse> listener) {
client.restoreSnapshot(request, listener); client.restoreSnapshot(request, listener);

View File

@ -73,7 +73,8 @@ public class TransportRestoreSnapshotAction extends TransportMasterNodeOperation
RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest( RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest(
"restore_snapshot[" + request.snapshot() + "]", request.repository(), request.snapshot(), "restore_snapshot[" + request.snapshot() + "]", request.repository(), request.snapshot(),
request.indices(), request.indicesOptions(), request.renamePattern(), request.renameReplacement(), request.indices(), request.indicesOptions(), request.renamePattern(), request.renameReplacement(),
request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases()); request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases(),
request.indexSettings(), request.ignoreIndexSettings());
restoreService.restoreSnapshot(restoreRequest, new ActionListener<RestoreInfo>() { restoreService.restoreSnapshot(restoreRequest, new ActionListener<RestoreInfo>() {
@Override @Override

View File

@ -24,6 +24,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.IndicesOptions;
@ -41,6 +42,7 @@ import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
@ -57,6 +59,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet; import static com.google.common.collect.Sets.newHashSet;
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
import static org.elasticsearch.cluster.metadata.MetaDataIndexStateService.INDEX_CLOSED_BLOCK; import static org.elasticsearch.cluster.metadata.MetaDataIndexStateService.INDEX_CLOSED_BLOCK;
/** /**
@ -85,6 +88,21 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
public static final String UPDATE_RESTORE_ACTION_NAME = "internal:cluster/snapshot/update_restore"; public static final String UPDATE_RESTORE_ACTION_NAME = "internal:cluster/snapshot/update_restore";
private static final ImmutableSet<String> UNMODIFIABLE_SETTINGS = ImmutableSet.of(
SETTING_NUMBER_OF_SHARDS,
SETTING_VERSION_CREATED,
SETTING_LEGACY_ROUTING_HASH_FUNCTION,
SETTING_LEGACY_ROUTING_USE_TYPE,
SETTING_UUID,
SETTING_CREATION_DATE);
// It's OK to change some settings, but we shouldn't allow simply removing them
private static final ImmutableSet<String> UNREMOVABLE_SETTINGS = ImmutableSet.<String>builder()
.addAll(UNMODIFIABLE_SETTINGS)
.add(SETTING_NUMBER_OF_REPLICAS)
.add(SETTING_AUTO_EXPAND_REPLICAS)
.build();
private final ClusterService clusterService; private final ClusterService clusterService;
private final RepositoriesService repositoriesService; private final RepositoriesService repositoriesService;
@ -163,6 +181,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
RestoreSource restoreSource = new RestoreSource(snapshotId, index); RestoreSource restoreSource = new RestoreSource(snapshotId, index);
String renamedIndex = indexEntry.getKey(); String renamedIndex = indexEntry.getKey();
IndexMetaData snapshotIndexMetaData = metaData.index(index); IndexMetaData snapshotIndexMetaData = metaData.index(index);
snapshotIndexMetaData = updateIndexSettings(snapshotIndexMetaData, request.indexSettings, request.ignoreIndexSettings);
// Check that the index is closed or doesn't exist // Check that the index is closed or doesn't exist
IndexMetaData currentIndexMetaData = currentState.metaData().index(renamedIndex); IndexMetaData currentIndexMetaData = currentState.metaData().index(renamedIndex);
IntSet ignoreShards = new IntOpenHashSet(); IntSet ignoreShards = new IntOpenHashSet();
@ -287,6 +306,51 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
} }
} }
/**
* Optionally updates index settings in indexMetaData by removing settings listed in ignoreSettings and
* merging them with settings in changeSettings.
*/
private IndexMetaData updateIndexSettings(IndexMetaData indexMetaData, Settings changeSettings, String[] ignoreSettings) {
if (changeSettings.names().isEmpty() && ignoreSettings.length == 0) {
return indexMetaData;
}
IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData);
Map<String, String> settingsMap = newHashMap(indexMetaData.settings().getAsMap());
List<String> simpleMatchPatterns = newArrayList();
for (String ignoredSetting : ignoreSettings) {
if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
throw new SnapshotRestoreException(snapshotId, "cannot remove setting [" + ignoredSetting + "] on restore");
} else {
settingsMap.remove(ignoredSetting);
}
} else {
simpleMatchPatterns.add(ignoredSetting);
}
}
if (!simpleMatchPatterns.isEmpty()) {
String[] removePatterns = simpleMatchPatterns.toArray(new String[simpleMatchPatterns.size()]);
Iterator<Map.Entry<String, String>> iterator = settingsMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String> entry = iterator.next();
if (UNREMOVABLE_SETTINGS.contains(entry.getKey()) == false) {
if (Regex.simpleMatch(removePatterns, entry.getKey())) {
iterator.remove();
}
}
}
}
for(Map.Entry<String, String> entry : changeSettings.getAsMap().entrySet()) {
if (UNMODIFIABLE_SETTINGS.contains(entry.getKey())) {
throw new SnapshotRestoreException(snapshotId, "cannot modify setting [" + entry.getKey() + "] on restore");
} else {
settingsMap.put(entry.getKey(), entry.getValue());
}
}
return builder.settings(ImmutableSettings.builder().put(settingsMap)).build();
}
private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) { private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) {
if (request.includeGlobalState()) { if (request.includeGlobalState()) {
if (metaData.persistentSettings() != null) { if (metaData.persistentSettings() != null) {
@ -703,6 +767,10 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
final private boolean includeAliases; final private boolean includeAliases;
final private Settings indexSettings;
final private String[] ignoreIndexSettings;
/** /**
* Constructs new restore request * Constructs new restore request
* *
@ -717,10 +785,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
* @param masterNodeTimeout master node timeout * @param masterNodeTimeout master node timeout
* @param includeGlobalState include global state into restore * @param includeGlobalState include global state into restore
* @param partial allow partial restore * @param partial allow partial restore
* @param indexSettings index settings that should be changed on restore
* @param ignoreIndexSettings index settings that shouldn't be restored
*/ */
public RestoreRequest(String cause, String repository, String name, String[] indices, IndicesOptions indicesOptions, public RestoreRequest(String cause, String repository, String name, String[] indices, IndicesOptions indicesOptions,
String renamePattern, String renameReplacement, Settings settings, String renamePattern, String renameReplacement, Settings settings,
TimeValue masterNodeTimeout, boolean includeGlobalState, boolean partial, boolean includeAliases) { TimeValue masterNodeTimeout, boolean includeGlobalState, boolean partial, boolean includeAliases,
Settings indexSettings, String[] ignoreIndexSettings ) {
this.cause = cause; this.cause = cause;
this.name = name; this.name = name;
this.repository = repository; this.repository = repository;
@ -733,6 +804,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
this.includeGlobalState = includeGlobalState; this.includeGlobalState = includeGlobalState;
this.partial = partial; this.partial = partial;
this.includeAliases = includeAliases; this.includeAliases = includeAliases;
this.indexSettings = indexSettings;
this.ignoreIndexSettings = ignoreIndexSettings;
} }
/** /**
@ -834,6 +908,25 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
return includeAliases; return includeAliases;
} }
/**
* Returns index settings that should be changed on restore
*
* @return restore aliases state flag
*/
public Settings indexSettings() {
return indexSettings;
}
/**
* Returns index settings that that shouldn't be restored
*
* @return restore aliases state flag
*/
public String[] ignoreIndexSettings() {
return ignoreIndexSettings;
}
/** /**
* Return master node timeout * Return master node timeout
* *

View File

@ -47,6 +47,7 @@ import org.elasticsearch.cluster.metadata.SnapshotMetaData;
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.store.support.AbstractIndexStore; import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.indices.InvalidIndexNameException;
@ -64,7 +65,9 @@ import java.util.concurrent.TimeUnit;
import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Lists.newArrayList;
import static org.elasticsearch.cluster.metadata.IndexMetaData.*; import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
import static org.elasticsearch.index.shard.IndexShard.*;
import static org.hamcrest.Matchers.*; import static org.hamcrest.Matchers.*;
@Slow @Slow
@ -77,7 +80,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)) .put("location", newTempDirPath())
.put("compress", randomBoolean()) .put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000)))); .put("chunk_size", randomIntBetween(100, 1000))));
@ -216,7 +219,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)) .put("location", newTempDirPath())
.put("compress", randomBoolean()) .put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000)))); .put("chunk_size", randomIntBetween(100, 1000))));
@ -451,7 +454,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings( .setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
ImmutableSettings.settingsBuilder() ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.TEST)) .put("location", newTempDirPath())
.put("random", randomAsciiOfLength(10)) .put("random", randomAsciiOfLength(10))
.put("random_control_io_exception_rate", 0.2)) .put("random_control_io_exception_rate", 0.2))
.setVerify(false)); .setVerify(false));
@ -501,7 +504,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings( .setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
ImmutableSettings.settingsBuilder() ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.TEST)) .put("location", newTempDirPath())
.put("random", randomAsciiOfLength(10)) .put("random", randomAsciiOfLength(10))
.put("random_data_file_io_exception_rate", 0.3))); .put("random_data_file_io_exception_rate", 0.3)));
@ -563,7 +566,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
@Test @Test
public void dataFileFailureDuringRestoreTest() throws Exception { public void dataFileFailureDuringRestoreTest() throws Exception {
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST); Path repositoryLocation = newTempDirPath();
Client client = client(); Client client = client();
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
@ -605,7 +608,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
@Test @Test
public void deletionOfFailingToRecoverIndexShouldStopRestore() throws Exception { public void deletionOfFailingToRecoverIndexShouldStopRestore() throws Exception {
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST); Path repositoryLocation = newTempDirPath();
Client client = client(); Client client = client();
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
@ -674,7 +677,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)))); .put("location", newTempDirPath())));
logger.info("--> creating index that cannot be allocated"); logger.info("--> creating index that cannot be allocated");
prepareCreate("test-idx", 2, ImmutableSettings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + ".tag", "nowhere").put("index.number_of_shards", 3)).get(); prepareCreate("test-idx", 2, ImmutableSettings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + ".tag", "nowhere").put("index.number_of_shards", 3)).get();
@ -692,7 +695,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
final int numberOfSnapshots = between(5, 15); final int numberOfSnapshots = between(5, 15);
Client client = client(); Client client = client();
Path repo = newTempDirPath(LifecycleScope.SUITE); Path repo = newTempDirPath();
logger.info("--> creating repository at " + repo.toAbsolutePath()); logger.info("--> creating repository at " + repo.toAbsolutePath());
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
@ -749,7 +752,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
public void deleteSnapshotWithMissingIndexAndShardMetadataTest() throws Exception { public void deleteSnapshotWithMissingIndexAndShardMetadataTest() throws Exception {
Client client = client(); Client client = client();
Path repo = newTempDirPath(LifecycleScope.SUITE); Path repo = newTempDirPath();
logger.info("--> creating repository at " + repo.toAbsolutePath()); logger.info("--> creating repository at " + repo.toAbsolutePath());
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
@ -788,7 +791,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
public void deleteSnapshotWithMissingMetadataTest() throws Exception { public void deleteSnapshotWithMissingMetadataTest() throws Exception {
Client client = client(); Client client = client();
Path repo = newTempDirPath(LifecycleScope.SUITE); Path repo = newTempDirPath();
logger.info("--> creating repository at " + repo.toAbsolutePath()); logger.info("--> creating repository at " + repo.toAbsolutePath());
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
@ -826,7 +829,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)))); .put("location", newTempDirPath())));
createIndex("test-idx", "test-idx-closed"); createIndex("test-idx", "test-idx-closed");
ensureGreen(); ensureGreen();
@ -852,7 +855,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)))); .put("location", newTempDirPath())));
createIndex("test-idx"); createIndex("test-idx");
ensureGreen(); ensureGreen();
@ -873,7 +876,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)))); .put("location", newTempDirPath())));
createIndex("test-idx-1", "test-idx-2", "test-idx-3"); createIndex("test-idx-1", "test-idx-2", "test-idx-3");
ensureGreen(); ensureGreen();
@ -989,7 +992,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
@Test @Test
public void moveShardWhileSnapshottingTest() throws Exception { public void moveShardWhileSnapshottingTest() throws Exception {
Client client = client(); Client client = client();
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST); Path repositoryLocation = newTempDirPath();
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings( .setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
@ -1051,7 +1054,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
@Test @Test
public void deleteRepositoryWhileSnapshottingTest() throws Exception { public void deleteRepositoryWhileSnapshottingTest() throws Exception {
Client client = client(); Client client = client();
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST); Path repositoryLocation = newTempDirPath();
logger.info("--> creating repository"); logger.info("--> creating repository");
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings( .setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
@ -1136,7 +1139,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
Client client = client(); Client client = client();
logger.info("--> creating repository"); logger.info("--> creating repository");
Path repositoryLocation = newTempDirPath(LifecycleScope.SUITE); Path repositoryLocation = newTempDirPath();
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", repositoryLocation) .put("location", repositoryLocation)
@ -1194,7 +1197,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
Client client = client(); Client client = client();
logger.info("--> creating repository"); logger.info("--> creating repository");
Path repositoryLocation = newTempDirPath(LifecycleScope.SUITE); Path repositoryLocation = newTempDirPath();
boolean throttleSnapshot = randomBoolean(); boolean throttleSnapshot = randomBoolean();
boolean throttleRestore = randomBoolean(); boolean throttleRestore = randomBoolean();
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
@ -1252,7 +1255,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
@Test @Test
public void snapshotStatusTest() throws Exception { public void snapshotStatusTest() throws Exception {
Client client = client(); Client client = client();
Path repositoryLocation = newTempDirPath(LifecycleScope.TEST); Path repositoryLocation = newTempDirPath();
logger.info("--> creating repository"); logger.info("--> creating repository");
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType(MockRepositoryModule.class.getCanonicalName()).setSettings( .setType(MockRepositoryModule.class.getCanonicalName()).setSettings(
@ -1347,7 +1350,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)) .put("location", newTempDirPath())
.put("compress", randomBoolean()) .put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000)))); .put("chunk_size", randomIntBetween(100, 1000))));
@ -1395,7 +1398,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> creating repository"); logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo") assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder() .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath(LifecycleScope.SUITE)) .put("location", newTempDirPath())
.put("compress", randomBoolean()) .put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000)))); .put("chunk_size", randomIntBetween(100, 1000))));
@ -1451,6 +1454,110 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
} }
} }
@Test
public void changeSettingsOnRestoreTest() throws Exception {
Client client = client();
logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
.put("location", newTempDirPath())
.put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000))));
logger.info("--> create test index with synonyms search analyzer");
ImmutableSettings.Builder indexSettings = ImmutableSettings.builder()
.put(indexSettings())
.put(SETTING_NUMBER_OF_REPLICAS, between(0, 1))
.put(INDEX_REFRESH_INTERVAL, "10s")
.put("index.analysis.analyzer.my_analyzer.type", "custom")
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
.putArray("index.analysis.analyzer.my_analyzer.filter", "lowercase", "my_synonym")
.put("index.analysis.filter.my_synonym.type", "synonym")
.put("index.analysis.filter.my_synonym.synonyms", "foo => bar");
assertAcked(prepareCreate("test-idx", 2, indexSettings));
int numberOfShards = getNumShards("test-idx").numPrimaries;
assertAcked(client().admin().indices().preparePutMapping("test-idx").setType("type1").setSource("field1", "type=string,search_analyzer=my_analyzer"));
final int numdocs = randomIntBetween(10, 100);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex("test-idx", "type1", Integer.toString(i)).setSource("field1", "bar " + i);
}
indexRandom(true, builders);
flushAndRefresh();
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), numdocs);
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs);
logger.info("--> snapshot it");
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
logger.info("--> delete the index and recreate it while changing refresh interval and analyzer");
cluster().wipeIndices("test-idx");
Settings newIndexSettings = ImmutableSettings.builder()
.put(INDEX_REFRESH_INTERVAL, "5s")
.put("index.analysis.analyzer.my_analyzer.type", "standard")
.build();
Settings newIncorrectIndexSettings = ImmutableSettings.builder()
.put(newIndexSettings)
.put(SETTING_NUMBER_OF_SHARDS, numberOfShards + 100)
.build();
logger.info("--> try restoring while changing the number of shards - should fail");
assertThrows(client.admin().cluster()
.prepareRestoreSnapshot("test-repo", "test-snap")
.setIgnoreIndexSettings("index.analysis.*")
.setIndexSettings(newIncorrectIndexSettings)
.setWaitForCompletion(true), SnapshotRestoreException.class);
logger.info("--> restore index with correct settings from the snapshot");
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster()
.prepareRestoreSnapshot("test-repo", "test-snap")
.setIgnoreIndexSettings("index.analysis.*")
.setIndexSettings(newIndexSettings)
.setWaitForCompletion(true).execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
logger.info("--> assert that correct settings are restored");
GetSettingsResponse getSettingsResponse = client.admin().indices().prepareGetSettings("test-idx").execute().actionGet();
assertThat(getSettingsResponse.getSetting("test-idx", INDEX_REFRESH_INTERVAL), equalTo("5s"));
// Make sure that number of shards didn't change
assertThat(getSettingsResponse.getSetting("test-idx", SETTING_NUMBER_OF_SHARDS), equalTo("" + numberOfShards));
assertThat(getSettingsResponse.getSetting("test-idx", "index.analysis.analyzer.my_analyzer.type"), equalTo("standard"));
assertThat(getSettingsResponse.getSetting("test-idx", "index.analysis.filter.my_synonym.type"), nullValue());
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), 0);
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs);
logger.info("--> delete the index and recreate it while deleting all index settings");
cluster().wipeIndices("test-idx");
logger.info("--> restore index with correct settings from the snapshot");
restoreSnapshotResponse = client.admin().cluster()
.prepareRestoreSnapshot("test-repo", "test-snap")
.setIgnoreIndexSettings("*") // delete everything we can delete
.setIndexSettings(newIndexSettings)
.setWaitForCompletion(true).execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
logger.info("--> assert that correct settings are restored and index is still functional");
getSettingsResponse = client.admin().indices().prepareGetSettings("test-idx").execute().actionGet();
assertThat(getSettingsResponse.getSetting("test-idx", INDEX_REFRESH_INTERVAL), equalTo("5s"));
// Make sure that number of shards didn't change
assertThat(getSettingsResponse.getSetting("test-idx", SETTING_NUMBER_OF_SHARDS), equalTo("" + numberOfShards));
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), 0);
assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs);
}
private boolean waitForIndex(final String index, TimeValue timeout) throws InterruptedException { private boolean waitForIndex(final String index, TimeValue timeout) throws InterruptedException {
return awaitBusy(new Predicate<Object>() { return awaitBusy(new Predicate<Object>() {
@Override @Override