honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add History UUID Index Setting (#56930) (#57104) 

Pre-requesite for #50278 to be able to uniquely identify index metadata by
its version fields and UUIDs when restoring into closed indices.
This commit is contained in:
Armin Braun 2020-05-25 11:26:03 +02:00 committed by GitHub
parent a4eb3edf46
commit 9fa60f7367
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 43 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.DiscoverySettings;
@ -47,6 +48,7 @@ import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVE
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase {
@ -293,6 +295,8 @@ public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase {
logger.info("--> ensure index test is green");
ensureGreen("test");
IndexMetadata indexMetadata = clusterService().state().metadata().index("test");
assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());
logger.info("--> detach-cluster on 2nd and 3rd master-eligible nodes");
Environment environmentMaster2 = TestEnvironment.newEnvironment(

4
server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.indices.recovery;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.test.ESIntegTestCase;
@ -31,6 +32,7 @@ import static org.elasticsearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMB
import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
@ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST)
public class DanglingIndicesIT extends ESIntegTestCase {
@ -89,6 +91,8 @@ public class DanglingIndicesIT extends ESIntegTestCase {
equalTo("42s"));
}
ensureGreen(INDEX_NAME);
final IndexMetadata indexMetadata = clusterService().state().metadata().index(INDEX_NAME);
assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());
}
/**

2
server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View File

@ -3597,6 +3597,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
final IndexMetadata indexMetadata = client().admin().cluster().prepareState().clear().setIndices(indexName)
.setMetadata(true).get().getState().metadata().index(indexName);
assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), nullValue());
final int numPrimaries = getNumShards(indexName).numPrimaries;
final Map<Integer, Long> primaryTerms = IntStream.range(0, numPrimaries)
.boxed().collect(Collectors.toMap(shardId -> shardId, indexMetadata::primaryTerm));
@ -3619,6 +3620,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
for (int shardId = 0; shardId < numPrimaries; shardId++) {
assertThat(restoredIndexMetadata.primaryTerm(shardId), greaterThan(primaryTerms.get(shardId)));
}
assertThat(restoredIndexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());
}
public void testSnapshotDifferentIndicesBySameName() {

6
server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
View File

@ -121,8 +121,10 @@ public class TransportResizeAction extends TransportMasterNodeAction<ResizeReque
if (metadata == null) {
throw new IndexNotFoundException(sourceIndexName);
}
final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings())
.normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
final Settings.Builder targetIndexSettingsBuilder = Settings.builder().put(targetIndex.settings())
.normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
targetIndexSettingsBuilder.remove(IndexMetadata.SETTING_HISTORY_UUID);
final Settings targetIndexSettings = targetIndexSettingsBuilder.build();
final int numShards;
if (IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
numShards = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings);

14
server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java
View File

@ -18,12 +18,15 @@
*/
package org.elasticsearch.cluster.coordination;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import joptsimple.OptionSet;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
@ -107,13 +110,18 @@ public class UnsafeBootstrapMasterCommand extends ElasticsearchNodeCommand {
.put(metadata.persistentSettings())
.put(UNSAFE_BOOTSTRAP.getKey(), true)
.build();
Metadata newMetadata = Metadata.builder(metadata)
Metadata.Builder newMetadata = Metadata.builder(metadata)
.clusterUUID(Metadata.UNKNOWN_CLUSTER_UUID)
.generateClusterUuidIfNeeded()
.clusterUUIDCommitted(true)
.persistentSettings(persistentSettings)
.coordinationMetadata(newCoordinationMetadata)
.build();
.coordinationMetadata(newCoordinationMetadata);
for (ObjectCursor<IndexMetadata> idx : metadata.indices().values()) {
IndexMetadata indexMetadata = idx.value;
newMetadata.put(IndexMetadata.builder(indexMetadata).settings(
Settings.builder().put(indexMetadata.getSettings())
.put(IndexMetadata.SETTING_HISTORY_UUID, UUIDs.randomBase64UUID())));
}
final ClusterState newClusterState = ClusterState.builder(oldClusterState)
.metadata(newMetadata).build();

1
server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
View File

@ -227,6 +227,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
Setting.intSetting("index.priority", 1, 0, Property.Dynamic, Property.IndexScope);
public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
public static final String SETTING_INDEX_UUID = "index.uuid";
public static final String SETTING_HISTORY_UUID = "index.history.uuid";
public static final String SETTING_DATA_PATH = "index.data_path";
public static final Setting<String> INDEX_DATA_PATH_SETTING =
new Setting<>(SETTING_DATA_PATH, "", Function.identity(), Property.IndexScope);

1
server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
View File

@ -214,6 +214,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
switch (key) {
case IndexMetadata.SETTING_CREATION_DATE:
case IndexMetadata.SETTING_INDEX_UUID:
case IndexMetadata.SETTING_HISTORY_UUID:
case IndexMetadata.SETTING_VERSION_UPGRADED:
case IndexMetadata.SETTING_INDEX_PROVIDED_NAME:
case MergePolicyConfig.INDEX_MERGE_ENABLED:

5
server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
View File

@ -35,9 +35,11 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.MasterNotDiscoveredException;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
@ -142,6 +144,9 @@ public class LocalAllocateDangledIndices {
// with the current version and upgrade it if needed.
upgradedIndexMetadata = metadataIndexUpgradeService.upgradeIndexMetadata(indexMetadata,
minIndexCompatibilityVersion);
upgradedIndexMetadata = IndexMetadata.builder(upgradedIndexMetadata).settings(
Settings.builder().put(upgradedIndexMetadata.getSettings()).put(
IndexMetadata.SETTING_HISTORY_UUID, UUIDs.randomBase64UUID())).build();
} catch (Exception ex) {
// upgrade failed - adding index as closed
logger.warn(() -> new ParameterizedMessage("found dangled index [{}] on node [{}]. This index cannot be " +

5
server/src/main/java/org/elasticsearch/index/IndexSettings.java
View File

@ -708,6 +708,11 @@ public final class IndexSettings {
if (newUUID.equals(getUUID()) == false) {
throw new IllegalArgumentException("uuid mismatch on settings update expected: " + getUUID() + " but was: " + newUUID);
}
final String newRestoreUUID = newSettings.get(IndexMetadata.SETTING_HISTORY_UUID, IndexMetadata.INDEX_UUID_NA_VALUE);
final String restoreUUID = this.settings.get(IndexMetadata.SETTING_HISTORY_UUID, IndexMetadata.INDEX_UUID_NA_VALUE);
if (newRestoreUUID.equals(restoreUUID) == false) {
throw new IllegalArgumentException("uuid mismatch on settings update expected: " + restoreUUID + " but was: " + newRestoreUUID);
}
this.indexMetadata = indexMetadata;
final Settings newIndexSettings = Settings.builder().put(nodeSettings).put(newSettings).build();
if (same(this.settings, newIndexSettings)) {

8
server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
View File

@ -91,6 +91,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
@ -125,7 +126,8 @@ public class RestoreService implements ClusterStateApplier {
SETTING_VERSION_CREATED,
SETTING_INDEX_UUID,
SETTING_CREATION_DATE,
IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey()));
IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
SETTING_HISTORY_UUID));
// It's OK to change some settings, but we shouldn't allow simply removing them
private static final Set<String> UNREMOVABLE_SETTINGS;
@ -342,8 +344,8 @@ public class RestoreService implements ClusterStateApplier {
}
indexMdBuilder.settings(Settings.builder()
.put(snapshotIndexMetadata.getSettings())
.put(IndexMetadata.SETTING_INDEX_UUID,
currentIndexMetadata.getIndexUUID()));
.put(IndexMetadata.SETTING_INDEX_UUID, currentIndexMetadata.getIndexUUID())
.put(IndexMetadata.SETTING_HISTORY_UUID, UUIDs.randomBase64UUID()));
IndexMetadata updatedIndexMetadata = indexMdBuilder.index(renamedIndexName).build();
rtBuilder.addAsRestore(updatedIndexMetadata, recoverySource);
blocks.updateBlocks(updatedIndexMetadata);

1
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
View File

@ -456,6 +456,7 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction<Resum
settings.remove(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey());
settings.remove(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey());
settings.remove(IndexMetadata.SETTING_INDEX_UUID);
settings.remove(IndexMetadata.SETTING_HISTORY_UUID);
settings.remove(IndexMetadata.SETTING_INDEX_PROVIDED_NAME);
settings.remove(IndexMetadata.SETTING_CREATION_DATE);