honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup Blobstore Repository Metadata Serialization (#62727) (#63249) 

Follow ups to #62684 making use of shorter utility for corruption checks.
This commit is contained in:
Armin Braun 2020-10-05 17:44:27 +02:00 committed by GitHub
parent 509fa46c9e
commit 89de9fdcf7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 95 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.index.store.StoreFileMetadata;
import java.io.IOException;
@ -509,36 +510,33 @@ public class BlobStoreIndexShardSnapshot implements ToXContentFragment {
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
token = parser.nextToken();
if (token.isValue()) {
if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) {
snapshot = parser.text();
} else if (PARSE_INDEX_VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
// The index-version is needed for backward compatibility with v 1.0
indexVersion = parser.longValue();
} else if (PARSE_START_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
startTime = parser.longValue();
} else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
time = parser.longValue();
} else if (PARSE_INCREMENTAL_FILE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) {
incrementalFileCount = parser.intValue();
} else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) {
incrementalSize = parser.longValue();
} else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) {
while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
indexFiles.add(FileInfo.fromXContent(parser));
}
} else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
final String currentFieldName = parser.currentName();
token = parser.nextToken();
if (token.isValue()) {
if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) {
snapshot = parser.text();
} else if (PARSE_INDEX_VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
// The index-version is needed for backward compatibility with v 1.0
indexVersion = parser.longValue();
} else if (PARSE_START_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
startTime = parser.longValue();
} else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
time = parser.longValue();
} else if (PARSE_INCREMENTAL_FILE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) {
incrementalFileCount = parser.intValue();
} else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) {
incrementalSize = parser.longValue();
} else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) {
while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
indexFiles.add(FileInfo.fromXContent(parser));
}
} else {
throw new ElasticsearchParseException("unexpected token [{}]", token);
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
}
} else {
throw new ElasticsearchParseException("unexpected token [{}]", token);

14
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
import java.io.IOException;
@ -237,9 +238,7 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
Map<String, FileInfo> files = new HashMap<>();
if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token != XContentParser.Token.FIELD_NAME) {
throw new ElasticsearchParseException("unexpected token [{}]", token);
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
String currentFieldName = parser.currentName();
token = parser.nextToken();
if (token == XContentParser.Token.START_ARRAY) {
@ -255,13 +254,10 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
}
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token != XContentParser.Token.FIELD_NAME) {
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
String snapshot = parser.currentName();
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(),
parser::getTokenLocation);
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();

30
server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
View File

@ -529,9 +529,7 @@ public final class RepositoryData {
* from cached bytes that we trust to not contain broken generations.
*/
public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticsearchParseException("start object expected");
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
final Map<String, SnapshotId> snapshots = new HashMap<>();
final Map<String, SnapshotState> snapshotStates = new HashMap<>();
@ -551,15 +549,13 @@ public final class RepositoryData {
parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations);
break;
case INDEX_METADATA_IDENTIFIERS:
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticsearchParseException("start object expected [" + INDEX_METADATA_IDENTIFIERS + "]");
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(),
parser::getTokenLocation);
indexMetaIdentifiers = parser.mapStrings();
break;
case MIN_VERSION:
if (parser.nextToken() != XContentParser.Token.VALUE_STRING) {
throw new ElasticsearchParseException("version string expected [min_version]");
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.nextToken(),
parser::getTokenLocation);
final Version version = Version.fromString(parser.text());
assert SnapshotsService.useShardGenerations(version);
break;
@ -615,9 +611,7 @@ public final class RepositoryData {
private static void parseSnapshots(XContentParser parser, Map<String, SnapshotId> snapshots, Map<String, SnapshotState> snapshotStates,
Map<String, Version> snapshotVersions,
Map<SnapshotId, Map<String, String>> indexMetaLookup) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_ARRAY) {
throw new ElasticsearchParseException("expected array for [" + SNAPSHOTS + "]");
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
String name = null;
String uuid = null;
@ -673,18 +667,14 @@ public final class RepositoryData {
private static void parseIndices(XContentParser parser, boolean fixBrokenShardGens, Map<String, SnapshotId> snapshots,
Map<IndexId, List<SnapshotId>> indexSnapshots, Map<String, IndexId> indexLookup,
ShardGenerations.Builder shardGenerations) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticsearchParseException("start object expected [indices]");
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
final String indexName = parser.currentName();
final List<SnapshotId> snapshotIds = new ArrayList<>();
final List<String> gens = new ArrayList<>();
IndexId indexId = null;
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticsearchParseException("start object expected index[" + indexName + "]");
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
final String indexMetaFieldName = parser.currentName();
final XContentParser.Token currentToken = parser.nextToken();
@ -693,9 +683,7 @@ public final class RepositoryData {
indexId = new IndexId(indexName, parser.text());
break;
case SNAPSHOTS:
if (currentToken != XContentParser.Token.START_ARRAY) {
throw new ElasticsearchParseException("start array expected [snapshots]");
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, currentToken, parser::getTokenLocation);
XContentParser.Token currToken;
while ((currToken = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
final String uuid;

113
server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.snapshots;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
@ -34,6 +33,7 @@ import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.rest.RestStatus;
@ -634,70 +634,67 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
parser.nextToken();
}
XContentParser.Token token;
if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) {
String currentFieldName = parser.currentName();
if (SNAPSHOT.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
token = parser.nextToken();
if (token.isValue()) {
if (NAME.equals(currentFieldName)) {
name = parser.text();
} else if (UUID.equals(currentFieldName)) {
uuid = parser.text();
} else if (STATE.equals(currentFieldName)) {
state = SnapshotState.valueOf(parser.text());
} else if (REASON.equals(currentFieldName)) {
reason = parser.text();
} else if (START_TIME.equals(currentFieldName)) {
startTime = parser.longValue();
} else if (END_TIME.equals(currentFieldName)) {
endTime = parser.longValue();
} else if (TOTAL_SHARDS.equals(currentFieldName)) {
totalShards = parser.intValue();
} else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) {
successfulShards = parser.intValue();
} else if (VERSION_ID.equals(currentFieldName)) {
version = Version.fromId(parser.intValue());
} else if (INCLUDE_GLOBAL_STATE.equals(currentFieldName)) {
includeGlobalState = parser.booleanValue();
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
String currentFieldName = parser.currentName();
if (SNAPSHOT.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
token = parser.nextToken();
if (token.isValue()) {
if (NAME.equals(currentFieldName)) {
name = parser.text();
} else if (UUID.equals(currentFieldName)) {
uuid = parser.text();
} else if (STATE.equals(currentFieldName)) {
state = SnapshotState.valueOf(parser.text());
} else if (REASON.equals(currentFieldName)) {
reason = parser.text();
} else if (START_TIME.equals(currentFieldName)) {
startTime = parser.longValue();
} else if (END_TIME.equals(currentFieldName)) {
endTime = parser.longValue();
} else if (TOTAL_SHARDS.equals(currentFieldName)) {
totalShards = parser.intValue();
} else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) {
successfulShards = parser.intValue();
} else if (VERSION_ID.equals(currentFieldName)) {
version = Version.fromId(parser.intValue());
} else if (INCLUDE_GLOBAL_STATE.equals(currentFieldName)) {
includeGlobalState = parser.booleanValue();
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (DATA_STREAMS.equals(currentFieldName)){
dataStreams = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
dataStreams.add(parser.text());
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (DATA_STREAMS.equals(currentFieldName)){
dataStreams = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
dataStreams.add(parser.text());
}
} else if (INDICES.equals(currentFieldName)) {
ArrayList<String> indicesArray = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
indicesArray.add(parser.text());
}
indices = Collections.unmodifiableList(indicesArray);
} else if (FAILURES.equals(currentFieldName)) {
ArrayList<SnapshotShardFailure> shardFailureArrayList = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser));
}
shardFailures = Collections.unmodifiableList(shardFailureArrayList);
} else {
// It was probably created by newer version - ignoring
parser.skipChildren();
} else if (INDICES.equals(currentFieldName)) {
ArrayList<String> indicesArray = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
indicesArray.add(parser.text());
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (USER_METADATA.equals(currentFieldName)) {
userMetadata = parser.map();
} else {
// It was probably created by newer version - ignoring
parser.skipChildren();
indices = Collections.unmodifiableList(indicesArray);
} else if (FAILURES.equals(currentFieldName)) {
ArrayList<SnapshotShardFailure> shardFailureArrayList = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser));
}
shardFailures = Collections.unmodifiableList(shardFailureArrayList);
} else {
// It was probably created by newer version - ignoring
parser.skipChildren();
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (USER_METADATA.equals(currentFieldName)) {
userMetadata = parser.map();
} else {
// It was probably created by newer version - ignoring
parser.skipChildren();
}
}
}
}
} else {
throw new ElasticsearchParseException("unexpected token [" + token + "]");
}
if (uuid == null) {
// the old format where there wasn't a UUID