* At least all the way back to 6.x we never use anything but `SMILE` in production code with this class so I removed the more general constructor and removed the format leniency from the deserialization
This commit is contained in:
parent
76a96c3774
commit
0cc94a457d
|
@ -72,15 +72,11 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
|
||||||
SNAPSHOT_ONLY_FORMAT_PARAMS = new ToXContent.MapParams(snapshotOnlyParams);
|
SNAPSHOT_ONLY_FORMAT_PARAMS = new ToXContent.MapParams(snapshotOnlyParams);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final XContentType DEFAULT_X_CONTENT_TYPE = XContentType.SMILE;
|
|
||||||
|
|
||||||
// The format version
|
// The format version
|
||||||
public static final int VERSION = 1;
|
public static final int VERSION = 1;
|
||||||
|
|
||||||
private static final int BUFFER_SIZE = 4096;
|
private static final int BUFFER_SIZE = 4096;
|
||||||
|
|
||||||
private final XContentType xContentType;
|
|
||||||
|
|
||||||
private final boolean compress;
|
private final boolean compress;
|
||||||
|
|
||||||
private final String codec;
|
private final String codec;
|
||||||
|
@ -91,23 +87,6 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
|
||||||
|
|
||||||
private final NamedXContentRegistry namedXContentRegistry;
|
private final NamedXContentRegistry namedXContentRegistry;
|
||||||
|
|
||||||
/**
|
|
||||||
* @param codec codec name
|
|
||||||
* @param blobNameFormat format of the blobname in {@link String#format} format
|
|
||||||
* @param reader prototype object that can deserialize T from XContent
|
|
||||||
* @param compress true if the content should be compressed
|
|
||||||
* @param xContentType content type that should be used for write operations
|
|
||||||
*/
|
|
||||||
public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
|
|
||||||
NamedXContentRegistry namedXContentRegistry, boolean compress, XContentType xContentType) {
|
|
||||||
this.reader = reader;
|
|
||||||
this.blobNameFormat = blobNameFormat;
|
|
||||||
this.namedXContentRegistry = namedXContentRegistry;
|
|
||||||
this.xContentType = xContentType;
|
|
||||||
this.compress = compress;
|
|
||||||
this.codec = codec;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param codec codec name
|
* @param codec codec name
|
||||||
* @param blobNameFormat format of the blobname in {@link String#format} format
|
* @param blobNameFormat format of the blobname in {@link String#format} format
|
||||||
|
@ -116,7 +95,11 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
|
||||||
*/
|
*/
|
||||||
public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
|
public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
|
||||||
NamedXContentRegistry namedXContentRegistry, boolean compress) {
|
NamedXContentRegistry namedXContentRegistry, boolean compress) {
|
||||||
this(codec, blobNameFormat, reader, namedXContentRegistry, compress, DEFAULT_X_CONTENT_TYPE);
|
this.reader = reader;
|
||||||
|
this.blobNameFormat = blobNameFormat;
|
||||||
|
this.namedXContentRegistry = namedXContentRegistry;
|
||||||
|
this.compress = compress;
|
||||||
|
this.codec = codec;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -158,7 +141,7 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
|
||||||
long filePointer = indexInput.getFilePointer();
|
long filePointer = indexInput.getFilePointer();
|
||||||
long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
|
long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
|
||||||
try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
|
try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
|
||||||
bytes.slice((int) filePointer, (int) contentSize))) {
|
bytes.slice((int) filePointer, (int) contentSize), XContentType.SMILE)) {
|
||||||
return reader.apply(parser);
|
return reader.apply(parser);
|
||||||
}
|
}
|
||||||
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
|
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
|
||||||
|
@ -237,7 +220,7 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void write(T obj, StreamOutput streamOutput) throws IOException {
|
private void write(T obj, StreamOutput streamOutput) throws IOException {
|
||||||
try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, streamOutput)) {
|
try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.SMILE, streamOutput)) {
|
||||||
builder.startObject();
|
builder.startObject();
|
||||||
obj.toXContent(builder, SNAPSHOT_ONLY_FORMAT_PARAMS);
|
obj.toXContent(builder, SNAPSHOT_ONLY_FORMAT_PARAMS);
|
||||||
builder.endObject();
|
builder.endObject();
|
||||||
|
|
|
@ -35,7 +35,6 @@ import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
import org.elasticsearch.common.xcontent.ToXContentFragment;
|
import org.elasticsearch.common.xcontent.ToXContentFragment;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.common.xcontent.XContentParser;
|
import org.elasticsearch.common.xcontent.XContentParser;
|
||||||
import org.elasticsearch.common.xcontent.XContentType;
|
|
||||||
import org.elasticsearch.index.translog.BufferedChecksumStreamOutput;
|
import org.elasticsearch.index.translog.BufferedChecksumStreamOutput;
|
||||||
import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat;
|
import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat;
|
||||||
import org.elasticsearch.snapshots.mockstore.BlobContainerWrapper;
|
import org.elasticsearch.snapshots.mockstore.BlobContainerWrapper;
|
||||||
|
@ -110,24 +109,17 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
|
||||||
public void testBlobStoreOperations() throws IOException {
|
public void testBlobStoreOperations() throws IOException {
|
||||||
BlobStore blobStore = createTestBlobStore();
|
BlobStore blobStore = createTestBlobStore();
|
||||||
BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
|
BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
|
||||||
ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
|
||||||
xContentRegistry(), false, XContentType.JSON);
|
|
||||||
ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
||||||
xContentRegistry(), false, XContentType.SMILE);
|
xContentRegistry(), false);
|
||||||
ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
||||||
xContentRegistry(), true, XContentType.SMILE);
|
xContentRegistry(), true);
|
||||||
|
|
||||||
// Write blobs in different formats
|
// Write blobs in different formats
|
||||||
checksumJSON.write(new BlobObj("checksum json"), blobContainer, "check-json");
|
|
||||||
checksumSMILE.write(new BlobObj("checksum smile"), blobContainer, "check-smile");
|
checksumSMILE.write(new BlobObj("checksum smile"), blobContainer, "check-smile");
|
||||||
checksumSMILECompressed.write(new BlobObj("checksum smile compressed"), blobContainer, "check-smile-comp");
|
checksumSMILECompressed.write(new BlobObj("checksum smile compressed"), blobContainer, "check-smile-comp");
|
||||||
|
|
||||||
// Assert that all checksum blobs can be read by all formats
|
// Assert that all checksum blobs can be read by all formats
|
||||||
assertEquals(checksumJSON.read(blobContainer, "check-json").getText(), "checksum json");
|
|
||||||
assertEquals(checksumSMILE.read(blobContainer, "check-json").getText(), "checksum json");
|
|
||||||
assertEquals(checksumJSON.read(blobContainer, "check-smile").getText(), "checksum smile");
|
|
||||||
assertEquals(checksumSMILE.read(blobContainer, "check-smile").getText(), "checksum smile");
|
assertEquals(checksumSMILE.read(blobContainer, "check-smile").getText(), "checksum smile");
|
||||||
assertEquals(checksumJSON.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
|
|
||||||
assertEquals(checksumSMILE.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
|
assertEquals(checksumSMILE.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -139,9 +131,9 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
|
||||||
veryRedundantText.append("Blah ");
|
veryRedundantText.append("Blah ");
|
||||||
}
|
}
|
||||||
ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
||||||
xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
|
xContentRegistry(), false);
|
||||||
ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
||||||
xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
|
xContentRegistry(), true);
|
||||||
BlobObj blobObj = new BlobObj(veryRedundantText.toString());
|
BlobObj blobObj = new BlobObj(veryRedundantText.toString());
|
||||||
checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
|
checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
|
||||||
checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
|
checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
|
||||||
|
@ -156,7 +148,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
|
||||||
String testString = randomAlphaOfLength(randomInt(10000));
|
String testString = randomAlphaOfLength(randomInt(10000));
|
||||||
BlobObj blobObj = new BlobObj(testString);
|
BlobObj blobObj = new BlobObj(testString);
|
||||||
ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
||||||
xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
|
xContentRegistry(), randomBoolean());
|
||||||
checksumFormat.write(blobObj, blobContainer, "test-path");
|
checksumFormat.write(blobObj, blobContainer, "test-path");
|
||||||
assertEquals(checksumFormat.read(blobContainer, "test-path").getText(), testString);
|
assertEquals(checksumFormat.read(blobContainer, "test-path").getText(), testString);
|
||||||
randomCorruption(blobContainer, "test-path");
|
randomCorruption(blobContainer, "test-path");
|
||||||
|
@ -191,7 +183,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
||||||
xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
|
xContentRegistry(), randomBoolean());
|
||||||
ExecutorService threadPool = Executors.newFixedThreadPool(1);
|
ExecutorService threadPool = Executors.newFixedThreadPool(1);
|
||||||
try {
|
try {
|
||||||
Future<Void> future = threadPool.submit(new Callable<Void>() {
|
Future<Void> future = threadPool.submit(new Callable<Void>() {
|
||||||
|
@ -215,7 +207,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
|
||||||
final String name = randomAlphaOfLength(10);
|
final String name = randomAlphaOfLength(10);
|
||||||
final BlobObj blobObj = new BlobObj("test");
|
final BlobObj blobObj = new BlobObj("test");
|
||||||
final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
|
||||||
xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
|
xContentRegistry(), randomBoolean());
|
||||||
|
|
||||||
final BlobStore blobStore = createTestBlobStore();
|
final BlobStore blobStore = createTestBlobStore();
|
||||||
final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
|
final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
|
||||||
|
|
Loading…
Reference in New Issue