honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup Blobstore Repository Metadata Serialization (#62727) (#63249) 

Follow ups to #62684 making use of shorter utility for corruption checks.
This commit is contained in:
Armin Braun 2020-10-05 17:44:27 +02:00 committed by GitHub
parent 509fa46c9e
commit 89de9fdcf7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 95 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.index.store.StoreFileMetadata; import org.elasticsearch.index.store.StoreFileMetadata;
import java.io.IOException; import java.io.IOException;
@ -509,36 +510,33 @@ public class BlobStoreIndexShardSnapshot implements ToXContentFragment {
XContentParser.Token token = parser.currentToken(); XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.START_OBJECT) { if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
String currentFieldName = parser.currentName(); final String currentFieldName = parser.currentName();
token = parser.nextToken(); token = parser.nextToken();
if (token.isValue()) { if (token.isValue()) {
if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) { if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) {
snapshot = parser.text(); snapshot = parser.text();
} else if (PARSE_INDEX_VERSION.match(currentFieldName, parser.getDeprecationHandler())) { } else if (PARSE_INDEX_VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
// The index-version is needed for backward compatibility with v 1.0 // The index-version is needed for backward compatibility with v 1.0
indexVersion = parser.longValue(); indexVersion = parser.longValue();
} else if (PARSE_START_TIME.match(currentFieldName, parser.getDeprecationHandler())) { } else if (PARSE_START_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
startTime = parser.longValue(); startTime = parser.longValue();
} else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) { } else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
time = parser.longValue(); time = parser.longValue();
} else if (PARSE_INCREMENTAL_FILE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) { } else if (PARSE_INCREMENTAL_FILE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) {
incrementalFileCount = parser.intValue(); incrementalFileCount = parser.intValue();
} else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { } else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) {
incrementalSize = parser.longValue(); incrementalSize = parser.longValue();
} else { } else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
} }
} else if (token == XContentParser.Token.START_ARRAY) { } else if (token == XContentParser.Token.START_ARRAY) {
if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) { if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) {
while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
indexFiles.add(FileInfo.fromXContent(parser)); indexFiles.add(FileInfo.fromXContent(parser));
}
} else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
} }
} else { } else {
throw new ElasticsearchParseException("unexpected token [{}]", token); throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
} }
} else { } else {
throw new ElasticsearchParseException("unexpected token [{}]", token); throw new ElasticsearchParseException("unexpected token [{}]", token);

14
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
import java.io.IOException; import java.io.IOException;
@ -237,9 +238,7 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
Map<String, FileInfo> files = new HashMap<>(); Map<String, FileInfo> files = new HashMap<>();
if (token == XContentParser.Token.START_OBJECT) { if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token != XContentParser.Token.FIELD_NAME) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
throw new ElasticsearchParseException("unexpected token [{}]", token);
}
String currentFieldName = parser.currentName(); String currentFieldName = parser.currentName();
token = parser.nextToken(); token = parser.nextToken();
if (token == XContentParser.Token.START_ARRAY) { if (token == XContentParser.Token.START_ARRAY) {
@ -255,13 +254,10 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName); throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
} }
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token != XContentParser.Token.FIELD_NAME) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
}
String snapshot = parser.currentName(); String snapshot = parser.currentName();
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(),
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName); parser::getTokenLocation);
}
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) { if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName(); currentFieldName = parser.currentName();

30
server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
View File

@ -529,9 +529,7 @@ public final class RepositoryData {
* from cached bytes that we trust to not contain broken generations. * from cached bytes that we trust to not contain broken generations.
*/ */
public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException { public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("start object expected");
}
final Map<String, SnapshotId> snapshots = new HashMap<>(); final Map<String, SnapshotId> snapshots = new HashMap<>();
final Map<String, SnapshotState> snapshotStates = new HashMap<>(); final Map<String, SnapshotState> snapshotStates = new HashMap<>();
@ -551,15 +549,13 @@ public final class RepositoryData {
parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations); parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations);
break; break;
case INDEX_METADATA_IDENTIFIERS: case INDEX_METADATA_IDENTIFIERS:
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(),
throw new ElasticsearchParseException("start object expected [" + INDEX_METADATA_IDENTIFIERS + "]"); parser::getTokenLocation);
}
indexMetaIdentifiers = parser.mapStrings(); indexMetaIdentifiers = parser.mapStrings();
break; break;
case MIN_VERSION: case MIN_VERSION:
if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.nextToken(),
throw new ElasticsearchParseException("version string expected [min_version]"); parser::getTokenLocation);
}
final Version version = Version.fromString(parser.text()); final Version version = Version.fromString(parser.text());
assert SnapshotsService.useShardGenerations(version); assert SnapshotsService.useShardGenerations(version);
break; break;
@ -615,9 +611,7 @@ public final class RepositoryData {
private static void parseSnapshots(XContentParser parser, Map<String, SnapshotId> snapshots, Map<String, SnapshotState> snapshotStates, private static void parseSnapshots(XContentParser parser, Map<String, SnapshotId> snapshots, Map<String, SnapshotState> snapshotStates,
Map<String, Version> snapshotVersions, Map<String, Version> snapshotVersions,
Map<SnapshotId, Map<String, String>> indexMetaLookup) throws IOException { Map<SnapshotId, Map<String, String>> indexMetaLookup) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_ARRAY) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("expected array for [" + SNAPSHOTS + "]");
}
while (parser.nextToken() != XContentParser.Token.END_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
String name = null; String name = null;
String uuid = null; String uuid = null;
@ -673,18 +667,14 @@ public final class RepositoryData {
private static void parseIndices(XContentParser parser, boolean fixBrokenShardGens, Map<String, SnapshotId> snapshots, private static void parseIndices(XContentParser parser, boolean fixBrokenShardGens, Map<String, SnapshotId> snapshots,
Map<IndexId, List<SnapshotId>> indexSnapshots, Map<String, IndexId> indexLookup, Map<IndexId, List<SnapshotId>> indexSnapshots, Map<String, IndexId> indexLookup,
ShardGenerations.Builder shardGenerations) throws IOException { ShardGenerations.Builder shardGenerations) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("start object expected [indices]");
}
while (parser.nextToken() != XContentParser.Token.END_OBJECT) { while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
final String indexName = parser.currentName(); final String indexName = parser.currentName();
final List<SnapshotId> snapshotIds = new ArrayList<>(); final List<SnapshotId> snapshotIds = new ArrayList<>();
final List<String> gens = new ArrayList<>(); final List<String> gens = new ArrayList<>();
IndexId indexId = null; IndexId indexId = null;
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("start object expected index[" + indexName + "]");
}
while (parser.nextToken() != XContentParser.Token.END_OBJECT) { while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
final String indexMetaFieldName = parser.currentName(); final String indexMetaFieldName = parser.currentName();
final XContentParser.Token currentToken = parser.nextToken(); final XContentParser.Token currentToken = parser.nextToken();
@ -693,9 +683,7 @@ public final class RepositoryData {
indexId = new IndexId(indexName, parser.text()); indexId = new IndexId(indexName, parser.text());
break; break;
case SNAPSHOTS: case SNAPSHOTS:
if (currentToken != XContentParser.Token.START_ARRAY) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, currentToken, parser::getTokenLocation);
throw new ElasticsearchParseException("start array expected [snapshots]");
}
XContentParser.Token currToken; XContentParser.Token currToken;
while ((currToken = parser.nextToken()) != XContentParser.Token.END_ARRAY) { while ((currToken = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
final String uuid; final String uuid;

113
server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
View File

@ -18,7 +18,6 @@
*/ */
package org.elasticsearch.snapshots; package org.elasticsearch.snapshots;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
@ -34,6 +33,7 @@ import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
@ -634,70 +634,67 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
parser.nextToken(); parser.nextToken();
} }
XContentParser.Token token; XContentParser.Token token;
if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
String currentFieldName = parser.currentName(); String currentFieldName = parser.currentName();
if (SNAPSHOT.equals(currentFieldName)) { if (SNAPSHOT.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) { if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName(); currentFieldName = parser.currentName();
token = parser.nextToken(); token = parser.nextToken();
if (token.isValue()) { if (token.isValue()) {
if (NAME.equals(currentFieldName)) { if (NAME.equals(currentFieldName)) {
name = parser.text(); name = parser.text();
} else if (UUID.equals(currentFieldName)) { } else if (UUID.equals(currentFieldName)) {
uuid = parser.text(); uuid = parser.text();
} else if (STATE.equals(currentFieldName)) { } else if (STATE.equals(currentFieldName)) {
state = SnapshotState.valueOf(parser.text()); state = SnapshotState.valueOf(parser.text());
} else if (REASON.equals(currentFieldName)) { } else if (REASON.equals(currentFieldName)) {
reason = parser.text(); reason = parser.text();
} else if (START_TIME.equals(currentFieldName)) { } else if (START_TIME.equals(currentFieldName)) {
startTime = parser.longValue(); startTime = parser.longValue();
} else if (END_TIME.equals(currentFieldName)) { } else if (END_TIME.equals(currentFieldName)) {
endTime = parser.longValue(); endTime = parser.longValue();
} else if (TOTAL_SHARDS.equals(currentFieldName)) { } else if (TOTAL_SHARDS.equals(currentFieldName)) {
totalShards = parser.intValue(); totalShards = parser.intValue();
} else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) { } else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) {
successfulShards = parser.intValue(); successfulShards = parser.intValue();
} else if (VERSION_ID.equals(currentFieldName)) { } else if (VERSION_ID.equals(currentFieldName)) {
version = Version.fromId(parser.intValue()); version = Version.fromId(parser.intValue());
} else if (INCLUDE_GLOBAL_STATE.equals(currentFieldName)) { } else if (INCLUDE_GLOBAL_STATE.equals(currentFieldName)) {
includeGlobalState = parser.booleanValue(); includeGlobalState = parser.booleanValue();
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (DATA_STREAMS.equals(currentFieldName)){
dataStreams = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
dataStreams.add(parser.text());
} }
} else if (token == XContentParser.Token.START_ARRAY) { } else if (INDICES.equals(currentFieldName)) {
if (DATA_STREAMS.equals(currentFieldName)){ ArrayList<String> indicesArray = new ArrayList<>();
dataStreams = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) { indicesArray.add(parser.text());
dataStreams.add(parser.text());
}
} else if (INDICES.equals(currentFieldName)) {
ArrayList<String> indicesArray = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
indicesArray.add(parser.text());
}
indices = Collections.unmodifiableList(indicesArray);
} else if (FAILURES.equals(currentFieldName)) {
ArrayList<SnapshotShardFailure> shardFailureArrayList = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser));
}
shardFailures = Collections.unmodifiableList(shardFailureArrayList);
} else {
// It was probably created by newer version - ignoring
parser.skipChildren();
} }
} else if (token == XContentParser.Token.START_OBJECT) { indices = Collections.unmodifiableList(indicesArray);
if (USER_METADATA.equals(currentFieldName)) { } else if (FAILURES.equals(currentFieldName)) {
userMetadata = parser.map(); ArrayList<SnapshotShardFailure> shardFailureArrayList = new ArrayList<>();
} else { while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
// It was probably created by newer version - ignoring shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser));
parser.skipChildren();
} }
shardFailures = Collections.unmodifiableList(shardFailureArrayList);
} else {
// It was probably created by newer version - ignoring
parser.skipChildren();
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (USER_METADATA.equals(currentFieldName)) {
userMetadata = parser.map();
} else {
// It was probably created by newer version - ignoring
parser.skipChildren();
} }
} }
} }
} }
} else {
throw new ElasticsearchParseException("unexpected token [" + token + "]");
} }
if (uuid == null) { if (uuid == null) {
// the old format where there wasn't a UUID // the old format where there wasn't a UUID