honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch to hardcoding Smile as the state format (#28610) 

This commit changes the state format that was previously passed in to
`MetaDataStateFormat` to always use Smile. This doesn't actually change the
format, since we have used Smile for writing the format since at least 5.0. This
removes the automatic detection of the state format when reading state, since
any state that could be processed in 6.x and 7.x would already have been written
in Smile format.

This is work towards removing the deprecated methods in the XContent code where
we do automatic content-type detection.

Relates to #28504
This commit is contained in:
Lee Hinman 2018-02-12 08:07:01 -07:00 committed by GitHub
parent c19d84012e
commit 6538542603
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 23 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View File

@ -1293,7 +1293,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, ToXContentFragmen
/**
* State format for {@link IndexMetaData} to write to and load from disk
*/
public static final MetaDataStateFormat<IndexMetaData> FORMAT = new MetaDataStateFormat<IndexMetaData>(XContentType.SMILE, INDEX_STATE_FILE_PREFIX) {
public static final MetaDataStateFormat<IndexMetaData> FORMAT = new MetaDataStateFormat<IndexMetaData>(INDEX_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {

2
server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View File

@ -1205,7 +1205,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
/**
* State format for {@link MetaData} to write to and load from disk
*/
public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(XContentType.SMILE, GLOBAL_STATE_FILE_PREFIX) {
public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {

2
server/src/main/java/org/elasticsearch/env/NodeMetaData.java vendored
View File

@ -91,7 +91,7 @@ public final class NodeMetaData {
}
public static final MetaDataStateFormat<NodeMetaData> FORMAT = new MetaDataStateFormat<NodeMetaData>(XContentType.SMILE, "node-") {
public static final MetaDataStateFormat<NodeMetaData> FORMAT = new MetaDataStateFormat<NodeMetaData>("node-") {
@Override
protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {

23
server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
View File

@ -64,36 +64,28 @@ import java.util.stream.Collectors;
* @param <T> the type of the XContent base data-structure
*/
public abstract class MetaDataStateFormat<T> {
public static final XContentType FORMAT = XContentType.SMILE;
public static final String STATE_DIR_NAME = "_state";
public static final String STATE_FILE_EXTENSION = ".st";
private static final String STATE_FILE_CODEC = "state";
private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 0;
private static final int STATE_FILE_VERSION = 1;
private static final int STATE_FILE_VERSION_ES_2X_AND_BELOW = 0;
private static final int BUFFER_SIZE = 4096;
private final XContentType format;
private final String prefix;
private final Pattern stateFilePattern;
/**
* Creates a new {@link MetaDataStateFormat} instance
* @param format the format of the x-content
*/
protected MetaDataStateFormat(XContentType format, String prefix) {
this.format = format;
protected MetaDataStateFormat(String prefix) {
this.prefix = prefix;
this.stateFilePattern = Pattern.compile(Pattern.quote(prefix) + "(\\d+)(" + MetaDataStateFormat.STATE_FILE_EXTENSION + ")?");
}
/**
* Returns the {@link XContentType} used to serialize xcontent on write.
*/
public XContentType format() {
return format;
}
/**
* Writes the given state to the given directories. The state is written to a
* state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it
@ -123,8 +115,8 @@ public abstract class MetaDataStateFormat<T> {
try (OutputStreamIndexOutput out =
new OutputStreamIndexOutput(resourceDesc, fileName, Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) {
CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION);
out.writeInt(format.index());
try (XContentBuilder builder = newXContentBuilder(format, new IndexOutputOutputStream(out) {
out.writeInt(FORMAT.index());
try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) {
@Override
public void close() throws IOException {
// this is important since some of the XContentBuilders write bytes on close.
@ -190,6 +182,9 @@ public abstract class MetaDataStateFormat<T> {
final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION,
STATE_FILE_VERSION);
final XContentType xContentType = XContentType.values()[indexInput.readInt()];
if (xContentType != FORMAT) {
throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType);
}
if (fileVersion == STATE_FILE_VERSION_ES_2X_AND_BELOW) {
// format version 0, wrote a version that always came from the content state file and was never used
indexInput.readLong(); // version currently unused
@ -197,7 +192,7 @@ public abstract class MetaDataStateFormat<T> {
long filePointer = indexInput.getFilePointer();
long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(namedXContentRegistry,
try (XContentParser parser = XContentFactory.xContent(FORMAT).createParser(namedXContentRegistry,
new InputStreamIndexInput(slice, contentSize))) {
return fromXContent(parser);
}

2
server/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
View File

@ -88,7 +88,7 @@ public final class ShardStateMetaData {
return "primary [" + primary + "], allocation [" + allocationId + "]";
}
public static final MetaDataStateFormat<ShardStateMetaData> FORMAT = new MetaDataStateFormat<ShardStateMetaData>(XContentType.SMILE, SHARD_STATE_FILE_PREFIX) {
public static final MetaDataStateFormat<ShardStateMetaData> FORMAT = new MetaDataStateFormat<ShardStateMetaData>(SHARD_STATE_FILE_PREFIX) {
@Override
protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {

21
server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
View File

@ -74,7 +74,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
* Ensure we can read a pre-generated cluster state.
*/
public void testReadClusterState() throws URISyntaxException, IOException {
final MetaDataStateFormat<MetaData> format = new MetaDataStateFormat<MetaData>(randomFrom(XContentType.values()), "global-") {
final MetaDataStateFormat<MetaData> format = new MetaDataStateFormat<MetaData>("global-") {
@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
@ -103,7 +103,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
dirs[i] = createTempDir();
}
final long id = addDummyFiles("foo-", dirs);
Format format = new Format(randomFrom(XContentType.values()), "foo-");
Format format = new Format("foo-");
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, dirs);
@ -145,7 +145,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
}
final long id = addDummyFiles("foo-", dirs);
Format format = new Format(randomFrom(XContentType.values()), "foo-");
Format format = new Format("foo-");
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, dirs);
@ -169,7 +169,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
dirs[i] = createTempDir();
}
final long id = addDummyFiles("foo-", dirs);
Format format = new Format(randomFrom(XContentType.values()), "foo-");
Format format = new Format("foo-");
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, dirs);
@ -244,17 +244,16 @@ public class MetaDataStateFormatTests extends ESTestCase {
meta.add(randomMeta());
}
Set<Path> corruptedFiles = new HashSet<>();
MetaDataStateFormat<MetaData> format = metaDataFormat(randomFrom(XContentType.values()));
MetaDataStateFormat<MetaData> format = metaDataFormat();
for (int i = 0; i < dirs.length; i++) {
dirs[i] = createTempDir();
Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
for (int j = 0; j < numLegacy; j++) {
XContentType type = format.format();
if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {
Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j);
Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them
} else {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type,
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(MetaDataStateFormat.FORMAT,
Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
xcontentBuilder.startObject();
MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, ToXContent.EMPTY_PARAMS);
@ -309,8 +308,8 @@ public class MetaDataStateFormatTests extends ESTestCase {
}
}
private static MetaDataStateFormat<MetaData> metaDataFormat(XContentType format) {
return new MetaDataStateFormat<MetaData>(format, MetaData.GLOBAL_STATE_FILE_PREFIX) {
private static MetaDataStateFormat<MetaData> metaDataFormat() {
return new MetaDataStateFormat<MetaData>(MetaData.GLOBAL_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
MetaData.Builder.toXContent(state, builder, ToXContent.EMPTY_PARAMS);
@ -347,8 +346,8 @@ public class MetaDataStateFormatTests extends ESTestCase {
private class Format extends MetaDataStateFormat<DummyState> {
Format(XContentType format, String prefix) {
super(format, prefix);
Format(String prefix) {
super(prefix);
}
@Override