HBASE-16993 BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
This commit is contained in:
Michael Stack
parent
287358ba6c
commit
de3a51263d
|
|
@ -0,0 +1,565 @@
|
|||
// Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
// source: BucketCache.proto
|
||||
|
||||
package org.apache.hadoop.hbase.shaded.protobuf.generated;
|
||||
|
||||
public final class BucketCacheProtos {
|
||||
private BucketCacheProtos() {}
|
||||
public static void registerAllExtensions(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite registry) {
|
||||
}
|
||||
|
||||
public static void registerAllExtensions(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistry registry) {
|
||||
registerAllExtensions(
|
||||
(org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite) registry);
|
||||
}
|
||||
public interface PersistedCacheMetadataOrBuilder extends
|
||||
// @@protoc_insertion_point(interface_extends:hbase.pb.PersistedCacheMetadata)
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
|
||||
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
boolean hasVersion();
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
int getVersion();
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
**
|
||||
* Metadata written out as preamble when we persist cache content.
|
||||
* </pre>
|
||||
*
|
||||
* Protobuf type {@code hbase.pb.PersistedCacheMetadata}
|
||||
*/
|
||||
public static final class PersistedCacheMetadata extends
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3 implements
|
||||
// @@protoc_insertion_point(message_implements:hbase.pb.PersistedCacheMetadata)
|
||||
PersistedCacheMetadataOrBuilder {
|
||||
// Use PersistedCacheMetadata.newBuilder() to construct.
|
||||
private PersistedCacheMetadata(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder<?> builder) {
|
||||
super(builder);
|
||||
}
|
||||
private PersistedCacheMetadata() {
|
||||
version_ = 0;
|
||||
}
|
||||
|
||||
@java.lang.Override
|
||||
public final org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
|
||||
getUnknownFields() {
|
||||
return this.unknownFields;
|
||||
}
|
||||
private PersistedCacheMetadata(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream input,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException {
|
||||
this();
|
||||
int mutable_bitField0_ = 0;
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder unknownFields =
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
|
||||
try {
|
||||
boolean done = false;
|
||||
while (!done) {
|
||||
int tag = input.readTag();
|
||||
switch (tag) {
|
||||
case 0:
|
||||
done = true;
|
||||
break;
|
||||
default: {
|
||||
if (!parseUnknownField(input, unknownFields,
|
||||
extensionRegistry, tag)) {
|
||||
done = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
bitField0_ |= 0x00000001;
|
||||
version_ = input.readUInt32();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException e) {
|
||||
throw e.setUnfinishedMessage(this);
|
||||
} catch (java.io.IOException e) {
|
||||
throw new org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
|
||||
e).setUnfinishedMessage(this);
|
||||
} finally {
|
||||
this.unknownFields = unknownFields.build();
|
||||
makeExtensionsImmutable();
|
||||
}
|
||||
}
|
||||
public static final org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
|
||||
getDescriptor() {
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.internal_static_hbase_pb_PersistedCacheMetadata_descriptor;
|
||||
}
|
||||
|
||||
protected org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
|
||||
internalGetFieldAccessorTable() {
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.internal_static_hbase_pb_PersistedCacheMetadata_fieldAccessorTable
|
||||
.ensureFieldAccessorsInitialized(
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata.class, org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata.Builder.class);
|
||||
}
|
||||
|
||||
private int bitField0_;
|
||||
public static final int VERSION_FIELD_NUMBER = 1;
|
||||
private int version_;
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
public boolean hasVersion() {
|
||||
return ((bitField0_ & 0x00000001) == 0x00000001);
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
public int getVersion() {
|
||||
return version_;
|
||||
}
|
||||
|
||||
private byte memoizedIsInitialized = -1;
|
||||
public final boolean isInitialized() {
|
||||
byte isInitialized = memoizedIsInitialized;
|
||||
if (isInitialized == 1) return true;
|
||||
if (isInitialized == 0) return false;
|
||||
|
||||
memoizedIsInitialized = 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
public void writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream output)
|
||||
throws java.io.IOException {
|
||||
if (((bitField0_ & 0x00000001) == 0x00000001)) {
|
||||
output.writeUInt32(1, version_);
|
||||
}
|
||||
unknownFields.writeTo(output);
|
||||
}
|
||||
|
||||
public int getSerializedSize() {
|
||||
int size = memoizedSize;
|
||||
if (size != -1) return size;
|
||||
|
||||
size = 0;
|
||||
if (((bitField0_ & 0x00000001) == 0x00000001)) {
|
||||
size += org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream
|
||||
.computeUInt32Size(1, version_);
|
||||
}
|
||||
size += unknownFields.getSerializedSize();
|
||||
memoizedSize = size;
|
||||
return size;
|
||||
}
|
||||
|
||||
private static final long serialVersionUID = 0L;
|
||||
@java.lang.Override
|
||||
public boolean equals(final java.lang.Object obj) {
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata)) {
|
||||
return super.equals(obj);
|
||||
}
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata other = (org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata) obj;
|
||||
|
||||
boolean result = true;
|
||||
result = result && (hasVersion() == other.hasVersion());
|
||||
if (hasVersion()) {
|
||||
result = result && (getVersion()
|
||||
== other.getVersion());
|
||||
}
|
||||
result = result && unknownFields.equals(other.unknownFields);
|
||||
return result;
|
||||
}
|
||||
|
||||
@java.lang.Override
|
||||
public int hashCode() {
|
||||
if (memoizedHashCode != 0) {
|
||||
return memoizedHashCode;
|
||||
}
|
||||
int hash = 41;
|
||||
hash = (19 * hash) + getDescriptorForType().hashCode();
|
||||
if (hasVersion()) {
|
||||
hash = (37 * hash) + VERSION_FIELD_NUMBER;
|
||||
hash = (53 * hash) + getVersion();
|
||||
}
|
||||
hash = (29 * hash) + unknownFields.hashCode();
|
||||
memoizedHashCode = hash;
|
||||
return hash;
|
||||
}
|
||||
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString data)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString data,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data, extensionRegistry);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(byte[] data)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(
|
||||
byte[] data,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data, extensionRegistry);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(java.io.InputStream input)
|
||||
throws java.io.IOException {
|
||||
return org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
|
||||
.parseWithIOException(PARSER, input);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(
|
||||
java.io.InputStream input,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
return org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
|
||||
.parseWithIOException(PARSER, input, extensionRegistry);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseDelimitedFrom(java.io.InputStream input)
|
||||
throws java.io.IOException {
|
||||
return org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
|
||||
.parseDelimitedWithIOException(PARSER, input);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseDelimitedFrom(
|
||||
java.io.InputStream input,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
return org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
|
||||
.parseDelimitedWithIOException(PARSER, input, extensionRegistry);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream input)
|
||||
throws java.io.IOException {
|
||||
return org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
|
||||
.parseWithIOException(PARSER, input);
|
||||
}
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parseFrom(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream input,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
return org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
|
||||
.parseWithIOException(PARSER, input, extensionRegistry);
|
||||
}
|
||||
|
||||
public Builder newBuilderForType() { return newBuilder(); }
|
||||
public static Builder newBuilder() {
|
||||
return DEFAULT_INSTANCE.toBuilder();
|
||||
}
|
||||
public static Builder newBuilder(org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata prototype) {
|
||||
return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
|
||||
}
|
||||
public Builder toBuilder() {
|
||||
return this == DEFAULT_INSTANCE
|
||||
? new Builder() : new Builder().mergeFrom(this);
|
||||
}
|
||||
|
||||
@java.lang.Override
|
||||
protected Builder newBuilderForType(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
|
||||
Builder builder = new Builder(parent);
|
||||
return builder;
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
**
|
||||
* Metadata written out as preamble when we persist cache content.
|
||||
* </pre>
|
||||
*
|
||||
* Protobuf type {@code hbase.pb.PersistedCacheMetadata}
|
||||
*/
|
||||
public static final class Builder extends
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder<Builder> implements
|
||||
// @@protoc_insertion_point(builder_implements:hbase.pb.PersistedCacheMetadata)
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadataOrBuilder {
|
||||
public static final org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
|
||||
getDescriptor() {
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.internal_static_hbase_pb_PersistedCacheMetadata_descriptor;
|
||||
}
|
||||
|
||||
protected org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
|
||||
internalGetFieldAccessorTable() {
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.internal_static_hbase_pb_PersistedCacheMetadata_fieldAccessorTable
|
||||
.ensureFieldAccessorsInitialized(
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata.class, org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata.Builder.class);
|
||||
}
|
||||
|
||||
// Construct using org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata.newBuilder()
|
||||
private Builder() {
|
||||
maybeForceBuilderInitialization();
|
||||
}
|
||||
|
||||
private Builder(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
|
||||
super(parent);
|
||||
maybeForceBuilderInitialization();
|
||||
}
|
||||
private void maybeForceBuilderInitialization() {
|
||||
if (org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
|
||||
.alwaysUseFieldBuilders) {
|
||||
}
|
||||
}
|
||||
public Builder clear() {
|
||||
super.clear();
|
||||
version_ = 0;
|
||||
bitField0_ = (bitField0_ & ~0x00000001);
|
||||
return this;
|
||||
}
|
||||
|
||||
public org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
|
||||
getDescriptorForType() {
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.internal_static_hbase_pb_PersistedCacheMetadata_descriptor;
|
||||
}
|
||||
|
||||
public org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata getDefaultInstanceForType() {
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata.getDefaultInstance();
|
||||
}
|
||||
|
||||
public org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata build() {
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata result = buildPartial();
|
||||
if (!result.isInitialized()) {
|
||||
throw newUninitializedMessageException(result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata buildPartial() {
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata result = new org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata(this);
|
||||
int from_bitField0_ = bitField0_;
|
||||
int to_bitField0_ = 0;
|
||||
if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
|
||||
to_bitField0_ |= 0x00000001;
|
||||
}
|
||||
result.version_ = version_;
|
||||
result.bitField0_ = to_bitField0_;
|
||||
onBuilt();
|
||||
return result;
|
||||
}
|
||||
|
||||
public Builder clone() {
|
||||
return (Builder) super.clone();
|
||||
}
|
||||
public Builder setField(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FieldDescriptor field,
|
||||
Object value) {
|
||||
return (Builder) super.setField(field, value);
|
||||
}
|
||||
public Builder clearField(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FieldDescriptor field) {
|
||||
return (Builder) super.clearField(field);
|
||||
}
|
||||
public Builder clearOneof(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.OneofDescriptor oneof) {
|
||||
return (Builder) super.clearOneof(oneof);
|
||||
}
|
||||
public Builder setRepeatedField(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FieldDescriptor field,
|
||||
int index, Object value) {
|
||||
return (Builder) super.setRepeatedField(field, index, value);
|
||||
}
|
||||
public Builder addRepeatedField(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FieldDescriptor field,
|
||||
Object value) {
|
||||
return (Builder) super.addRepeatedField(field, value);
|
||||
}
|
||||
public Builder mergeFrom(org.apache.hadoop.hbase.shaded.com.google.protobuf.Message other) {
|
||||
if (other instanceof org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata) {
|
||||
return mergeFrom((org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata)other);
|
||||
} else {
|
||||
super.mergeFrom(other);
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
public Builder mergeFrom(org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata other) {
|
||||
if (other == org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata.getDefaultInstance()) return this;
|
||||
if (other.hasVersion()) {
|
||||
setVersion(other.getVersion());
|
||||
}
|
||||
this.mergeUnknownFields(other.unknownFields);
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
|
||||
public final boolean isInitialized() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public Builder mergeFrom(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream input,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata parsedMessage = null;
|
||||
try {
|
||||
parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
|
||||
} catch (org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException e) {
|
||||
parsedMessage = (org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata) e.getUnfinishedMessage();
|
||||
throw e.unwrapIOException();
|
||||
} finally {
|
||||
if (parsedMessage != null) {
|
||||
mergeFrom(parsedMessage);
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
private int bitField0_;
|
||||
|
||||
private int version_ ;
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
public boolean hasVersion() {
|
||||
return ((bitField0_ & 0x00000001) == 0x00000001);
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
public int getVersion() {
|
||||
return version_;
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
public Builder setVersion(int value) {
|
||||
bitField0_ |= 0x00000001;
|
||||
version_ = value;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
* Set version to be zero
|
||||
* </pre>
|
||||
*
|
||||
* <code>optional uint32 version = 1 [default = 0];</code>
|
||||
*/
|
||||
public Builder clearVersion() {
|
||||
bitField0_ = (bitField0_ & ~0x00000001);
|
||||
version_ = 0;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
public final Builder setUnknownFields(
|
||||
final org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet unknownFields) {
|
||||
return super.setUnknownFields(unknownFields);
|
||||
}
|
||||
|
||||
public final Builder mergeUnknownFields(
|
||||
final org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet unknownFields) {
|
||||
return super.mergeUnknownFields(unknownFields);
|
||||
}
|
||||
|
||||
|
||||
// @@protoc_insertion_point(builder_scope:hbase.pb.PersistedCacheMetadata)
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(class_scope:hbase.pb.PersistedCacheMetadata)
|
||||
private static final org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata DEFAULT_INSTANCE;
|
||||
static {
|
||||
DEFAULT_INSTANCE = new org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata();
|
||||
}
|
||||
|
||||
public static org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata getDefaultInstance() {
|
||||
return DEFAULT_INSTANCE;
|
||||
}
|
||||
|
||||
@java.lang.Deprecated public static final org.apache.hadoop.hbase.shaded.com.google.protobuf.Parser<PersistedCacheMetadata>
|
||||
PARSER = new org.apache.hadoop.hbase.shaded.com.google.protobuf.AbstractParser<PersistedCacheMetadata>() {
|
||||
public PersistedCacheMetadata parsePartialFrom(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream input,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException {
|
||||
return new PersistedCacheMetadata(input, extensionRegistry);
|
||||
}
|
||||
};
|
||||
|
||||
public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Parser<PersistedCacheMetadata> parser() {
|
||||
return PARSER;
|
||||
}
|
||||
|
||||
@java.lang.Override
|
||||
public org.apache.hadoop.hbase.shaded.com.google.protobuf.Parser<PersistedCacheMetadata> getParserForType() {
|
||||
return PARSER;
|
||||
}
|
||||
|
||||
public org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos.PersistedCacheMetadata getDefaultInstanceForType() {
|
||||
return DEFAULT_INSTANCE;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static final org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
|
||||
internal_static_hbase_pb_PersistedCacheMetadata_descriptor;
|
||||
private static final
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
|
||||
internal_static_hbase_pb_PersistedCacheMetadata_fieldAccessorTable;
|
||||
|
||||
public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor
|
||||
getDescriptor() {
|
||||
return descriptor;
|
||||
}
|
||||
private static org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor
|
||||
descriptor;
|
||||
static {
|
||||
java.lang.String[] descriptorData = {
|
||||
"\n\021BucketCache.proto\022\010hbase.pb\",\n\026Persist" +
|
||||
"edCacheMetadata\022\022\n\007version\030\001 \001(\r:\0010BK\n1o" +
|
||||
"rg.apache.hadoop.hbase.shaded.protobuf.g" +
|
||||
"eneratedB\021BucketCacheProtosH\001\240\001\001"
|
||||
};
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
|
||||
new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() {
|
||||
public org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistry assignDescriptors(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor root) {
|
||||
descriptor = root;
|
||||
return null;
|
||||
}
|
||||
};
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor
|
||||
.internalBuildGeneratedFileFrom(descriptorData,
|
||||
new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor[] {
|
||||
}, assigner);
|
||||
internal_static_hbase_pb_PersistedCacheMetadata_descriptor =
|
||||
getDescriptor().getMessageTypes().get(0);
|
||||
internal_static_hbase_pb_PersistedCacheMetadata_fieldAccessorTable = new
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
|
||||
internal_static_hbase_pb_PersistedCacheMetadata_descriptor,
|
||||
new java.lang.String[] { "Version", });
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(outer_class_scope)
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// This file contains protocol buffers that are shared throughout HBase
|
||||
package hbase.pb;
|
||||
|
||||
option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated";
|
||||
option java_outer_classname = "BucketCacheProtos";
|
||||
option java_generate_equals_and_hash = true;
|
||||
option optimize_for = SPEED;
|
||||
|
||||
/**
|
||||
* Metadata written out as preamble when we persist cache content.
|
||||
*/
|
||||
message PersistedCacheMetadata {
|
||||
// Set version to be zero
|
||||
optional uint32 version = 1 [default = 0];
|
||||
}
|
||||
|
|
@ -352,6 +352,10 @@
|
|||
<groupId>org.apache.hbase</groupId>
|
||||
<artifactId>hbase-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hbase</groupId>
|
||||
<artifactId>hbase-protocol-shaded</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hbase</groupId>
|
||||
<artifactId>hbase-protocol</artifactId>
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.hbase.HBaseIOException;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.io.HeapSize;
|
||||
import org.apache.hadoop.hbase.io.hfile.BlockCache;
|
||||
|
|
@ -68,6 +69,9 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager;
|
|||
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
|
||||
import org.apache.hadoop.hbase.nio.ByteBuff;
|
||||
import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||
import org.apache.hadoop.hbase.util.HasThread;
|
||||
import org.apache.hadoop.hbase.util.IdReadWriteLock;
|
||||
|
|
@ -248,9 +252,11 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
try {
|
||||
retrieveFromFile(bucketSizes);
|
||||
} catch (IOException ioex) {
|
||||
LOG.error("Can't restore from file because of", ioex);
|
||||
LOG.error("Can't restore cache from persisted file " + persistencePath +
|
||||
"; file removed; cache is cold!", ioex);
|
||||
} catch (ClassNotFoundException cnfe) {
|
||||
LOG.error("Can't restore from file in rebuild because can't deserialise",cnfe);
|
||||
LOG.error("Can't restore cache from persisted file in rebuild "
|
||||
+ "because can't deserialise; file removed; cache is cold!", cnfe);
|
||||
throw new RuntimeException(cnfe);
|
||||
}
|
||||
}
|
||||
|
|
@ -945,6 +951,11 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
return receptacle;
|
||||
}
|
||||
|
||||
/**
|
||||
* The current version of the persisted cache file.
|
||||
*/
|
||||
private static final int PERSISTED_CACHE_VERSION = 0;
|
||||
|
||||
private void persistToFile() throws IOException {
|
||||
assert !cacheEnabled;
|
||||
FileOutputStream fos = null;
|
||||
|
|
@ -954,6 +965,13 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
throw new IOException("Attempt to persist non-persistent cache mappings!");
|
||||
}
|
||||
fos = new FileOutputStream(persistencePath, false);
|
||||
// Write out a metdata protobuf block in case we change format at later date, etc.
|
||||
// Add our magic as preamble.
|
||||
fos.write(ProtobufMagic.PB_MAGIC, 0, ProtobufMagic.lengthOfPBMagic());
|
||||
BucketCacheProtos.PersistedCacheMetadata metadata =
|
||||
BucketCacheProtos.PersistedCacheMetadata.newBuilder().
|
||||
setVersion(PERSISTED_CACHE_VERSION).build();
|
||||
metadata.writeDelimitedTo(fos);
|
||||
oos = new ObjectOutputStream(fos);
|
||||
oos.writeLong(cacheCapacity);
|
||||
oos.writeUTF(ioEngine.getClass().getName());
|
||||
|
|
@ -966,9 +984,12 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @see #persistToFile()
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private void retrieveFromFile(int[] bucketSizes) throws IOException, BucketAllocatorException,
|
||||
ClassNotFoundException {
|
||||
private void retrieveFromFile(int[] bucketSizes)
|
||||
throws IOException, BucketAllocatorException, ClassNotFoundException {
|
||||
File persistenceFile = new File(persistencePath);
|
||||
if (!persistenceFile.exists()) {
|
||||
return;
|
||||
|
|
@ -977,10 +998,35 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
FileInputStream fis = null;
|
||||
ObjectInputStream ois = null;
|
||||
try {
|
||||
if (!ioEngine.isPersistent())
|
||||
throw new IOException(
|
||||
"Attempt to restore non-persistent cache mappings!");
|
||||
if (!ioEngine.isPersistent()) {
|
||||
throw new IOException("Attempt to restore non-persistent cache mappings!");
|
||||
}
|
||||
fis = new FileInputStream(persistencePath);
|
||||
// Read protobuf magic and then metadata. See persistToFile for where we wrote
|
||||
// out metadata for format.
|
||||
byte [] pbmagic = new byte [ProtobufMagic.lengthOfPBMagic()];
|
||||
int len = fis.read(pbmagic, 0, pbmagic.length);
|
||||
if (len != pbmagic.length || !ProtobufMagic.isPBMagicPrefix(pbmagic)) {
|
||||
// Throw exception. In finally we remove the file ALWAYS.
|
||||
throw new HBaseIOException("Failed read of protobuf magic ("
|
||||
+ Bytes.toString(pbmagic)+ "); old format (HBASE-16993)? "
|
||||
+ "Failed read of persisted cache file=" + persistencePath);
|
||||
}
|
||||
BucketCacheProtos.PersistedCacheMetadata metadata = null;
|
||||
try {
|
||||
metadata =
|
||||
BucketCacheProtos.PersistedCacheMetadata.parseDelimitedFrom(fis);
|
||||
} catch (IOException e) {
|
||||
// Throw exception if failed parse. In finally we remove the
|
||||
throw new HBaseIOException("Failed read of persisted cache metadata file=" +
|
||||
persistencePath, e);
|
||||
}
|
||||
if (metadata.getVersion() != PERSISTED_CACHE_VERSION) {
|
||||
throw new HBaseIOException("Unexpected version of persisted cache metadata file=" +
|
||||
persistencePath + "; expected=" + PERSISTED_CACHE_VERSION + " but read=" +
|
||||
metadata.getVersion());
|
||||
}
|
||||
// Ok. Read metadata. All seems good. Go ahead and pull in the persisted cache.
|
||||
ois = new ObjectInputStream(fis);
|
||||
long capacitySize = ois.readLong();
|
||||
if (capacitySize != cacheCapacity)
|
||||
|
|
@ -1010,6 +1056,8 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
if (!persistenceFile.delete()) {
|
||||
throw new IOException("Failed deleting persistence file "
|
||||
+ persistenceFile.getAbsolutePath());
|
||||
} else {
|
||||
LOG.info("Deleted persisted cache file " + persistencePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1130,10 +1178,7 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
/**
|
||||
* Item in cache. We expect this to be where most memory goes. Java uses 8
|
||||
* bytes just for object headers; after this, we want to use as little as
|
||||
* possible - so we only use 8 bytes, but in order to do so we end up messing
|
||||
* around with all this Java casting stuff. Offset stored as 5 bytes that make
|
||||
* up the long. Doubt we'll see devices this big for ages. Offsets are divided
|
||||
* by 256. So 5 bytes gives us 256TB or so.
|
||||
* possible.
|
||||
*/
|
||||
static class BucketEntry implements Serializable {
|
||||
private static final long serialVersionUID = -6741504807982257534L;
|
||||
|
|
@ -1147,15 +1192,14 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
}
|
||||
};
|
||||
|
||||
private int offsetBase;
|
||||
private int length;
|
||||
private byte offset1;
|
||||
byte deserialiserIndex;
|
||||
private volatile long accessCounter;
|
||||
private BlockPriority priority;
|
||||
// Set this when we were not able to forcefully evict the block
|
||||
private volatile boolean markedForEvict;
|
||||
private AtomicInteger refCount = new AtomicInteger(0);
|
||||
private long offset;
|
||||
|
||||
/**
|
||||
* Time this block was cached. Presumes we are created just before we are added to the cache.
|
||||
|
|
@ -1173,17 +1217,12 @@ public class BucketCache implements BlockCache, HeapSize {
|
|||
}
|
||||
}
|
||||
|
||||
long offset() { // Java has no unsigned numbers
|
||||
long o = ((long) offsetBase) & 0xFFFFFFFF;
|
||||
o += (((long) (offset1)) & 0xFF) << 32;
|
||||
return o << 8;
|
||||
long offset() {
|
||||
return this.offset;
|
||||
}
|
||||
|
||||
private void setOffset(long value) {
|
||||
assert (value & 0xFF) == 0;
|
||||
value >>= 8;
|
||||
offsetBase = (int) value;
|
||||
offset1 = (byte) (value >> 32);
|
||||
this.offset = value;
|
||||
}
|
||||
|
||||
public int getLength() {
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.io.hfile.bucket;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
|
|
@ -60,12 +59,11 @@ public class TestBucketCache {
|
|||
@Parameterized.Parameters(name = "{index}: blockSize={0}, bucketSizes={1}")
|
||||
public static Iterable<Object[]> data() {
|
||||
return Arrays.asList(new Object[][] {
|
||||
{ 8192, null }, // TODO: why is 8k the default blocksize for these tests?
|
||||
{
|
||||
16 * 1024,
|
||||
new int[] { 2 * 1024 + 1024, 4 * 1024 + 1024, 8 * 1024 + 1024, 16 * 1024 + 1024,
|
||||
28 * 1024 + 1024, 32 * 1024 + 1024, 64 * 1024 + 1024, 96 * 1024 + 1024,
|
||||
128 * 1024 + 1024 } } });
|
||||
{44600, null}, // Random size here and below to demo no need for multiple of 256 (HBASE-16993)
|
||||
{16 * 1024,
|
||||
new int[] { 2 * 1024 + 1024, 4 * 1024 + 1024, 15000, 46000, 49152, 51200, 8 * 1024 + 1024,
|
||||
16 * 1024 + 1024, 28 * 1024 + 1024, 32 * 1024 + 1024, 64 * 1024 + 1024, 96 * 1024 + 1024,
|
||||
128 * 1024 + 1024 } } });
|
||||
}
|
||||
|
||||
@Parameterized.Parameter(0)
|
||||
|
|
@ -229,10 +227,11 @@ public class TestBucketCache {
|
|||
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
Path testDir = TEST_UTIL.getDataTestDir();
|
||||
TEST_UTIL.getTestFileSystem().mkdirs(testDir);
|
||||
|
||||
BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
|
||||
constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, testDir
|
||||
+ "/bucket.persistence");
|
||||
Path persistFile = new Path(testDir, "bucketcache.persist.file");
|
||||
String persistFileStr = persistFile.toString();
|
||||
String engine = "file:/" + persistFile.toString();
|
||||
BucketCache bucketCache = new BucketCache(engine, capacitySize,
|
||||
constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistFileStr);
|
||||
long usedSize = bucketCache.getAllocator().getUsedSize();
|
||||
assertTrue(usedSize == 0);
|
||||
|
||||
|
|
@ -246,26 +245,28 @@ public class TestBucketCache {
|
|||
}
|
||||
usedSize = bucketCache.getAllocator().getUsedSize();
|
||||
assertTrue(usedSize != 0);
|
||||
// persist cache to file
|
||||
// Persist cache to file
|
||||
bucketCache.shutdown();
|
||||
|
||||
// restore cache from file
|
||||
bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
|
||||
constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, testDir
|
||||
+ "/bucket.persistence");
|
||||
// Restore cache from file
|
||||
bucketCache = new BucketCache(engine, capacitySize, constructedBlockSize,
|
||||
constructedBlockSizes, writeThreads, writerQLen, persistFileStr);
|
||||
assertEquals(usedSize, bucketCache.getAllocator().getUsedSize());
|
||||
// Assert file is no longer present.
|
||||
assertFalse(TEST_UTIL.getTestFileSystem().exists(persistFile));
|
||||
// persist cache to file
|
||||
bucketCache.shutdown();
|
||||
|
||||
// reconfig buckets sizes, the biggest bucket is small than constructedBlockSize (8k or 16k)
|
||||
// so it can't restore cache from file
|
||||
int[] smallBucketSizes = new int[] { 2 * 1024 + 1024, 4 * 1024 + 1024 };
|
||||
bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
|
||||
constructedBlockSize, smallBucketSizes, writeThreads,
|
||||
writerQLen, testDir + "/bucket.persistence");
|
||||
// Throw in a few random sizes to demo don't have to be multiple of 256 (HBASE-16993)
|
||||
int[] smallBucketSizes = new int[] {2 * 1024 + 1024, 3600, 4 * 1024 + 1024, 47000 };
|
||||
bucketCache = new BucketCache(engine, capacitySize, constructedBlockSize, smallBucketSizes,
|
||||
writeThreads, writerQLen, persistFileStr);
|
||||
assertEquals(0, bucketCache.getAllocator().getUsedSize());
|
||||
assertEquals(0, bucketCache.backingMap.size());
|
||||
|
||||
// The above should have failed reading he cache file and then cleaned it up.
|
||||
assertFalse(TEST_UTIL.getTestFileSystem().exists(persistFile));
|
||||
TEST_UTIL.cleanupTestDir();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue