honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove non-SMILE Serialization from ChecksumBlobStoreFormat (#44278) (#44326) 

* At least all the way back to 6.x we never use anything but `SMILE` in
production code with this class so I removed the more general
constructor and removed the format leniency from the deserialization
This commit is contained in:
Armin Braun 2019-07-15 10:59:33 +02:00 committed by GitHub
parent 76a96c3774
commit 0cc94a457d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
View File

@ -72,15 +72,11 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
SNAPSHOT_ONLY_FORMAT_PARAMS = new ToXContent.MapParams(snapshotOnlyParams);
}
private static final XContentType DEFAULT_X_CONTENT_TYPE = XContentType.SMILE;
// The format version
public static final int VERSION = 1;
private static final int BUFFER_SIZE = 4096;
private final XContentType xContentType;
private final boolean compress;
private final String codec;
@ -91,23 +87,6 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
private final NamedXContentRegistry namedXContentRegistry;
/**
* @param codec codec name
* @param blobNameFormat format of the blobname in {@link String#format} format
* @param reader prototype object that can deserialize T from XContent
* @param compress true if the content should be compressed
* @param xContentType content type that should be used for write operations
*/
public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
NamedXContentRegistry namedXContentRegistry, boolean compress, XContentType xContentType) {
this.reader = reader;
this.blobNameFormat = blobNameFormat;
this.namedXContentRegistry = namedXContentRegistry;
this.xContentType = xContentType;
this.compress = compress;
this.codec = codec;
}
/**
* @param codec codec name
* @param blobNameFormat format of the blobname in {@link String#format} format
@ -116,7 +95,11 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
*/
public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
NamedXContentRegistry namedXContentRegistry, boolean compress) {
this(codec, blobNameFormat, reader, namedXContentRegistry, compress, DEFAULT_X_CONTENT_TYPE);
this.reader = reader;
this.blobNameFormat = blobNameFormat;
this.namedXContentRegistry = namedXContentRegistry;
this.compress = compress;
this.codec = codec;
}
/**
@ -158,7 +141,7 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
long filePointer = indexInput.getFilePointer();
long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
bytes.slice((int) filePointer, (int) contentSize))) {
bytes.slice((int) filePointer, (int) contentSize), XContentType.SMILE)) {
return reader.apply(parser);
}
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
@ -237,7 +220,7 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
}
private void write(T obj, StreamOutput streamOutput) throws IOException {
try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, streamOutput)) {
try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.SMILE, streamOutput)) {
builder.startObject();
obj.toXContent(builder, SNAPSHOT_ONLY_FORMAT_PARAMS);
builder.endObject();

22
server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.translog.BufferedChecksumStreamOutput;
import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat;
import org.elasticsearch.snapshots.mockstore.BlobContainerWrapper;
@ -110,24 +109,17 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
public void testBlobStoreOperations() throws IOException {
BlobStore blobStore = createTestBlobStore();
BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), false, XContentType.JSON);
ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), false, XContentType.SMILE);
xContentRegistry(), false);
ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), true, XContentType.SMILE);
xContentRegistry(), true);
// Write blobs in different formats
checksumJSON.write(new BlobObj("checksum json"), blobContainer, "check-json");
checksumSMILE.write(new BlobObj("checksum smile"), blobContainer, "check-smile");
checksumSMILECompressed.write(new BlobObj("checksum smile compressed"), blobContainer, "check-smile-comp");
// Assert that all checksum blobs can be read by all formats
assertEquals(checksumJSON.read(blobContainer, "check-json").getText(), "checksum json");
assertEquals(checksumSMILE.read(blobContainer, "check-json").getText(), "checksum json");
assertEquals(checksumJSON.read(blobContainer, "check-smile").getText(), "checksum smile");
assertEquals(checksumSMILE.read(blobContainer, "check-smile").getText(), "checksum smile");
assertEquals(checksumJSON.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
assertEquals(checksumSMILE.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
}
@ -139,9 +131,9 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
veryRedundantText.append("Blah ");
}
ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
xContentRegistry(), false);
ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
xContentRegistry(), true);
BlobObj blobObj = new BlobObj(veryRedundantText.toString());
checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
@ -156,7 +148,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
String testString = randomAlphaOfLength(randomInt(10000));
BlobObj blobObj = new BlobObj(testString);
ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
xContentRegistry(), randomBoolean());
checksumFormat.write(blobObj, blobContainer, "test-path");
assertEquals(checksumFormat.read(blobContainer, "test-path").getText(), testString);
randomCorruption(blobContainer, "test-path");
@ -191,7 +183,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
}
};
final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
xContentRegistry(), randomBoolean());
ExecutorService threadPool = Executors.newFixedThreadPool(1);
try {
Future<Void> future = threadPool.submit(new Callable<Void>() {
@ -215,7 +207,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
final String name = randomAlphaOfLength(10);
final BlobObj blobObj = new BlobObj("test");
final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
xContentRegistry(), randomBoolean());
final BlobStore blobStore = createTestBlobStore();
final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());