honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup Blobstore Repository Metadata Serialization (#62727) (#63249) 

Follow ups to #62684 making use of shorter utility for corruption checks.
This commit is contained in:
Armin Braun 2020-10-05 17:44:27 +02:00 committed by GitHub
parent 509fa46c9e
commit 89de9fdcf7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 95 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.index.store.StoreFileMetadata; import org.elasticsearch.index.store.StoreFileMetadata;
import java.io.IOException; import java.io.IOException;
@ -509,8 +510,8 @@ public class BlobStoreIndexShardSnapshot implements ToXContentFragment {
XContentParser.Token token = parser.currentToken(); XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.START_OBJECT) { if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
String currentFieldName = parser.currentName(); final String currentFieldName = parser.currentName();
token = parser.nextToken(); token = parser.nextToken();
if (token.isValue()) { if (token.isValue()) {
if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) { if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) {
@ -540,9 +541,6 @@ public class BlobStoreIndexShardSnapshot implements ToXContentFragment {
} else { } else {
throw new ElasticsearchParseException("unexpected token [{}]", token); throw new ElasticsearchParseException("unexpected token [{}]", token);
} }
} else {
throw new ElasticsearchParseException("unexpected token [{}]", token);
}
} }
} }

14
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
import java.io.IOException; import java.io.IOException;
@ -237,9 +238,7 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
Map<String, FileInfo> files = new HashMap<>(); Map<String, FileInfo> files = new HashMap<>();
if (token == XContentParser.Token.START_OBJECT) { if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token != XContentParser.Token.FIELD_NAME) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
throw new ElasticsearchParseException("unexpected token [{}]", token);
}
String currentFieldName = parser.currentName(); String currentFieldName = parser.currentName();
token = parser.nextToken(); token = parser.nextToken();
if (token == XContentParser.Token.START_ARRAY) { if (token == XContentParser.Token.START_ARRAY) {
@ -255,13 +254,10 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName); throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
} }
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token != XContentParser.Token.FIELD_NAME) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
}
String snapshot = parser.currentName(); String snapshot = parser.currentName();
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(),
throw new ElasticsearchParseException("unknown object [{}]", currentFieldName); parser::getTokenLocation);
}
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) { if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName(); currentFieldName = parser.currentName();

30
server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
View File

@ -529,9 +529,7 @@ public final class RepositoryData {
* from cached bytes that we trust to not contain broken generations. * from cached bytes that we trust to not contain broken generations.
*/ */
public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException { public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("start object expected");
}
final Map<String, SnapshotId> snapshots = new HashMap<>(); final Map<String, SnapshotId> snapshots = new HashMap<>();
final Map<String, SnapshotState> snapshotStates = new HashMap<>(); final Map<String, SnapshotState> snapshotStates = new HashMap<>();
@ -551,15 +549,13 @@ public final class RepositoryData {
parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations); parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations);
break; break;
case INDEX_METADATA_IDENTIFIERS: case INDEX_METADATA_IDENTIFIERS:
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(),
throw new ElasticsearchParseException("start object expected [" + INDEX_METADATA_IDENTIFIERS + "]"); parser::getTokenLocation);
}
indexMetaIdentifiers = parser.mapStrings(); indexMetaIdentifiers = parser.mapStrings();
break; break;
case MIN_VERSION: case MIN_VERSION:
if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.nextToken(),
throw new ElasticsearchParseException("version string expected [min_version]"); parser::getTokenLocation);
}
final Version version = Version.fromString(parser.text()); final Version version = Version.fromString(parser.text());
assert SnapshotsService.useShardGenerations(version); assert SnapshotsService.useShardGenerations(version);
break; break;
@ -615,9 +611,7 @@ public final class RepositoryData {
private static void parseSnapshots(XContentParser parser, Map<String, SnapshotId> snapshots, Map<String, SnapshotState> snapshotStates, private static void parseSnapshots(XContentParser parser, Map<String, SnapshotId> snapshots, Map<String, SnapshotState> snapshotStates,
Map<String, Version> snapshotVersions, Map<String, Version> snapshotVersions,
Map<SnapshotId, Map<String, String>> indexMetaLookup) throws IOException { Map<SnapshotId, Map<String, String>> indexMetaLookup) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_ARRAY) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("expected array for [" + SNAPSHOTS + "]");
}
while (parser.nextToken() != XContentParser.Token.END_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
String name = null; String name = null;
String uuid = null; String uuid = null;
@ -673,18 +667,14 @@ public final class RepositoryData {
private static void parseIndices(XContentParser parser, boolean fixBrokenShardGens, Map<String, SnapshotId> snapshots, private static void parseIndices(XContentParser parser, boolean fixBrokenShardGens, Map<String, SnapshotId> snapshots,
Map<IndexId, List<SnapshotId>> indexSnapshots, Map<String, IndexId> indexLookup, Map<IndexId, List<SnapshotId>> indexSnapshots, Map<String, IndexId> indexLookup,
ShardGenerations.Builder shardGenerations) throws IOException { ShardGenerations.Builder shardGenerations) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("start object expected [indices]");
}
while (parser.nextToken() != XContentParser.Token.END_OBJECT) { while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
final String indexName = parser.currentName(); final String indexName = parser.currentName();
final List<SnapshotId> snapshotIds = new ArrayList<>(); final List<SnapshotId> snapshotIds = new ArrayList<>();
final List<String> gens = new ArrayList<>(); final List<String> gens = new ArrayList<>();
IndexId indexId = null; IndexId indexId = null;
if (parser.nextToken() != XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
throw new ElasticsearchParseException("start object expected index[" + indexName + "]");
}
while (parser.nextToken() != XContentParser.Token.END_OBJECT) { while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
final String indexMetaFieldName = parser.currentName(); final String indexMetaFieldName = parser.currentName();
final XContentParser.Token currentToken = parser.nextToken(); final XContentParser.Token currentToken = parser.nextToken();
@ -693,9 +683,7 @@ public final class RepositoryData {
indexId = new IndexId(indexName, parser.text()); indexId = new IndexId(indexName, parser.text());
break; break;
case SNAPSHOTS: case SNAPSHOTS:
if (currentToken != XContentParser.Token.START_ARRAY) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, currentToken, parser::getTokenLocation);
throw new ElasticsearchParseException("start array expected [snapshots]");
}
XContentParser.Token currToken; XContentParser.Token currToken;
while ((currToken = parser.nextToken()) != XContentParser.Token.END_ARRAY) { while ((currToken = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
final String uuid; final String uuid;

7
server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
View File

@ -18,7 +18,6 @@
*/ */
package org.elasticsearch.snapshots; package org.elasticsearch.snapshots;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
@ -34,6 +33,7 @@ import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
@ -634,7 +634,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
parser.nextToken(); parser.nextToken();
} }
XContentParser.Token token; XContentParser.Token token;
if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
String currentFieldName = parser.currentName(); String currentFieldName = parser.currentName();
if (SNAPSHOT.equals(currentFieldName)) { if (SNAPSHOT.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@ -696,9 +696,6 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
} }
} }
} }
} else {
throw new ElasticsearchParseException("unexpected token [" + token + "]");
}
if (uuid == null) { if (uuid == null) {
// the old format where there wasn't a UUID // the old format where there wasn't a UUID
uuid = name; uuid = name;