honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-31 04:18:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

[7.x] Include hidden indices in snapshots by default ()

Browse Source 
Previously, hidden indices were not included in snapshots by default, unless
specified using one of the usual methods for doing so: naming indices directly,
using index patterns starting with a ., or specifying expand_wildcards to
a value that includes hidden (e.g. all or hidden,open).

This commit changes the default expand_wildcards value to include hidden
indices.
This commit is contained in:
Gordon Brown 2020-06-09 16:01:52 -06:00 committed by GitHub
parent 9eec819c5b
commit aab6317260
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 183 additions and 2 deletions
client/rest-high-level/src/test/java/org/elasticsearch/client
SnapshotIT.java
server/src
internalClusterTest/java/org/elasticsearch/snapshots
SharedClusterSnapshotRestoreIT.java
main/java/org/elasticsearch/action
admin/cluster/snapshots/create
CreateSnapshotRequest.java
support
IndicesOptions.java

43
client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java

@ -38,6 +38,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.repositories.fs.FsRepository;
@ -286,6 +287,48 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
assertThat(restoreInfo.failedShards(), equalTo(0));
}
public void testSnapshotHidden() throws IOException {
String testRepository = "test";
String testSnapshot = "snapshot_1";
String testIndex = "test_index";
AcknowledgedResponse putRepositoryResponse = createTestRepository(testRepository, FsRepository.TYPE, "{\"location\": \".\"}");
assertTrue(putRepositoryResponse.isAcknowledged());
createIndex(testIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.build());
assertTrue("index [" + testIndex + "] should have been created", indexExists(testIndex));
CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot);
createSnapshotRequest.indices("*");
createSnapshotRequest.waitForCompletion(true);
if (randomBoolean()) {
createSnapshotRequest.userMetadata(randomUserMetadata());
}
CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest);
assertEquals(RestStatus.OK, createSnapshotResponse.status());
deleteIndex(testIndex);
assertFalse("index [" + testIndex + "] should have been deleted", indexExists(testIndex));
RestoreSnapshotRequest request = new RestoreSnapshotRequest(testRepository, testSnapshot);
request.waitForCompletion(true);
request.indices(randomFrom(testIndex, "test_*"));
request.renamePattern(testIndex);
RestoreSnapshotResponse response = execute(request, highLevelClient().snapshot()::restore,
highLevelClient().snapshot()::restoreAsync);
RestoreInfo restoreInfo = response.getRestoreInfo();
assertThat(restoreInfo.name(), equalTo(testSnapshot));
assertThat(restoreInfo.indices(), equalTo(Collections.singletonList(testIndex)));
assertThat(restoreInfo.successfulShards(), greaterThan(0));
assertThat(restoreInfo.failedShards(), equalTo(0));
}
public void testDeleteSnapshot() throws IOException {
String repository = "test_repository";
String snapshot = "test_snapshot";

129
server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java

@ -133,6 +133,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertRequestBuilderThrows;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
@ -3734,6 +3735,134 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
assertThat(getSnapshotsResponse.getSnapshots(), empty());
}
public void testHiddenIndicesIncludedInSnapshot() {
Client client = client();
final String normalIndex = "normal-index";
final String hiddenIndex = "hidden-index";
final String dottedHiddenIndex = ".index-hidden";
final String repoName = "test-repo";
logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository(repoName).setType("fs").setSettings(randomRepoSettings()));
logger.info("--> creating indices");
createIndex(normalIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.build());
createIndex(hiddenIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.build());
createIndex(dottedHiddenIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.build());
ensureGreen();
logger.info("--> indexing some data");
for (int i = 0; i < 100; i++) {
index(normalIndex, "_doc", Integer.toString(i), "foo", "bar" + i);
index(hiddenIndex, "_doc", Integer.toString(i), "foo", "baz" + i);
index(dottedHiddenIndex, "_doc", Integer.toString(i), "foo", "baz" + i);
}
refresh();
assertHitCount(client.prepareSearch(normalIndex).setSize(0).get(), 100L);
assertHitCount(client.prepareSearch(hiddenIndex).setSize(0).get(), 100L);
assertHitCount(client.prepareSearch(dottedHiddenIndex).setSize(0).get(), 100L);
logger.info("--> taking a snapshot");
final String snapName = "test-snap";
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, snapName)
.setWaitForCompletion(true).setIndices(randomFrom("*", "_all")).get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
List<SnapshotInfo> snapshotInfos = client.admin().cluster().prepareGetSnapshots(repoName)
.setSnapshots(randomFrom(snapName, "_all", "*", "*-snap", "test*")).get().getSnapshots();
assertThat(snapshotInfos.size(), equalTo(1));
SnapshotInfo snapshotInfo = snapshotInfos.get(0);
assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
assertThat(snapshotInfo.version(), equalTo(Version.CURRENT));
logger.info("--> deleting indices");
cluster().wipeIndices(normalIndex, hiddenIndex, dottedHiddenIndex);
// Verify that hidden indices get restored with a wildcard restore
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices("*")
.execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(normalIndex, hiddenIndex, dottedHiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(true));
cluster().wipeIndices(normalIndex, hiddenIndex, dottedHiddenIndex);
}
// Verify that exclusions work on hidden indices
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices("*", "-.*")
.execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(normalIndex, hiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(false));
cluster().wipeIndices(normalIndex, hiddenIndex);
}
// Verify that hidden indices can be restored with a non-star pattern
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices("hid*")
.execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(hiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(false));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(false));
cluster().wipeIndices(hiddenIndex);
}
// Verify that hidden indices can be restored by fully specified name
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices(dottedHiddenIndex)
.get();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(dottedHiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(false));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(false));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(true));
}
}
private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map<String, List<String>> indicesPerSnapshot) {
for (SnapshotInfo snapshotInfo : response.getSnapshots()) {
final List<String> expected = snapshotInfo.indices();

3
server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java

@ -72,8 +72,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
private String repository;
private String[] indices = EMPTY_ARRAY;
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenHidden();
private boolean partial = false;

10
server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java

@ -109,6 +109,8 @@ public class IndicesOptions implements ToXContentFragment {
public static final IndicesOptions STRICT_EXPAND_OPEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN));
public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN, WildcardStates.HIDDEN));
public static final IndicesOptions LENIENT_EXPAND_OPEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
EnumSet.of(WildcardStates.OPEN));
@ -386,6 +388,14 @@ public class IndicesOptions implements ToXContentFragment {
return STRICT_EXPAND_OPEN;
}
/**
* @return indices options that requires every specified index to exist, expands wildcards only to open indices, includes hidden
* indices, and allows that no indices are resolved from wildcard expressions (not returning an error).
*/
public static IndicesOptions strictExpandOpenHidden() {
return STRICT_EXPAND_OPEN_HIDDEN;
}
/**
* @return indices options that requires every specified index to exist, expands wildcards only to open indices,
* allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the