HBASE-26025 Add a flag to mark if the IOError can be solved by retry in thrift IOError (#3414)

Signed-off-by: Reid Chan <reidchan@apache.org>

hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftHBaseServiceHandler.java

@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.CellBuilder;
 import org.apache.hadoop.hbase.CellBuilderFactory;
 import org.apache.hadoop.hbase.CellBuilderType;
 import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.KeyValue;
@@ -1291,6 +1292,7 @@ public class ThriftHBaseServiceHandler extends HBaseServiceHandler implements Hb
 
   private static IOError getIOError(Throwable throwable) {
     IOError error = new IOErrorWithCause(throwable);
+    error.setCanRetry(!(throwable instanceof DoNotRetryIOException));
     error.setMessage(Throwables.getStackTraceAsString(throwable));
     return error;
   }

hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java

@@ -17,15 +17,18 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IOError");
 
   private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
+  private static final org.apache.thrift.protocol.TField CAN_RETRY_FIELD_DESC = new org.apache.thrift.protocol.TField("canRetry", org.apache.thrift.protocol.TType.BOOL, (short)2);
 
   private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new IOErrorStandardSchemeFactory();
   private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new IOErrorTupleSchemeFactory();
 
   public @org.apache.thrift.annotation.Nullable java.lang.String message; // required
+  public boolean canRetry; // required
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    MESSAGE((short)1, "message");
+    MESSAGE((short)1, "message"),
+    CAN_RETRY((short)2, "canRetry");
 
     private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
 
@@ -43,6 +46,8 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
       switch(fieldId) {
         case 1: // MESSAGE
           return MESSAGE;
+        case 2: // CAN_RETRY
+          return CAN_RETRY;
         default:
           return null;
       }
@@ -84,11 +89,15 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
   }
 
   // isset id assignments
+  private static final int __CANRETRY_ISSET_ID = 0;
+  private byte __isset_bitfield = 0;
   public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
     tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
       new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+    tmpMap.put(_Fields.CAN_RETRY, new org.apache.thrift.meta_data.FieldMetaData("canRetry", org.apache.thrift.TFieldRequirementType.DEFAULT,
+      new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
     metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(IOError.class, metaDataMap);
   }
@@ -97,19 +106,24 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
   }
 
   public IOError(
-    java.lang.String message)
+    java.lang.String message,
+    boolean canRetry)
   {
     this();
     this.message = message;
+    this.canRetry = canRetry;
+    setCanRetryIsSet(true);
   }
 
   /**
    * Performs a deep copy on <i>other</i>.
    */
   public IOError(IOError other) {
+    __isset_bitfield = other.__isset_bitfield;
     if (other.isSetMessage()) {
       this.message = other.message;
     }
+    this.canRetry = other.canRetry;
   }
 
   public IOError deepCopy() {
@@ -119,6 +133,8 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
   @Override
   public void clear() {
     this.message = null;
+    setCanRetryIsSet(false);
+    this.canRetry = false;
   }
 
   @org.apache.thrift.annotation.Nullable
@@ -146,6 +162,29 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
     }
   }
 
+  public boolean isCanRetry() {
+    return this.canRetry;
+  }
+
+  public IOError setCanRetry(boolean canRetry) {
+    this.canRetry = canRetry;
+    setCanRetryIsSet(true);
+    return this;
+  }
+
+  public void unsetCanRetry() {
+    __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __CANRETRY_ISSET_ID);
+  }
+
+  /** Returns true if field canRetry is set (has been assigned a value) and false otherwise */
+  public boolean isSetCanRetry() {
+    return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __CANRETRY_ISSET_ID);
+  }
+
+  public void setCanRetryIsSet(boolean value) {
+    __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __CANRETRY_ISSET_ID, value);
+  }
+
   public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
     switch (field) {
       case MESSAGE:
@@ -156,6 +195,14 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
         }
         break;
 
+      case CAN_RETRY:
+        if (value == null) {
+          unsetCanRetry();
+        } else {
+          setCanRetry((java.lang.Boolean)value);
+        }
+        break;
+
     }
   }
 
@@ -165,6 +212,9 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
       case MESSAGE:
         return getMessage();
 
+      case CAN_RETRY:
+        return isCanRetry();
+
     }
     throw new java.lang.IllegalStateException();
   }
@@ -178,6 +228,8 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
     switch (field) {
       case MESSAGE:
         return isSetMessage();
+      case CAN_RETRY:
+        return isSetCanRetry();
     }
     throw new java.lang.IllegalStateException();
   }
@@ -204,6 +256,15 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
         return false;
     }
 
+    boolean this_present_canRetry = true;
+    boolean that_present_canRetry = true;
+    if (this_present_canRetry || that_present_canRetry) {
+      if (!(this_present_canRetry && that_present_canRetry))
+        return false;
+      if (this.canRetry != that.canRetry)
+        return false;
+    }
+
     return true;
   }
 
@@ -215,6 +276,8 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
     if (isSetMessage())
       hashCode = hashCode * 8191 + message.hashCode();
 
+    hashCode = hashCode * 8191 + ((canRetry) ? 131071 : 524287);
+
     return hashCode;
   }
 
@@ -236,6 +299,16 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
         return lastComparison;
       }
     }
+    lastComparison = java.lang.Boolean.compare(isSetCanRetry(), other.isSetCanRetry());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetCanRetry()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.canRetry, other.canRetry);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -264,6 +337,10 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
       sb.append(this.message);
     }
     first = false;
+    if (!first) sb.append(", ");
+    sb.append("canRetry:");
+    sb.append(this.canRetry);
+    first = false;
     sb.append(")");
     return sb.toString();
   }
@@ -283,6 +360,8 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
 
   private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException {
     try {
+      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+      __isset_bitfield = 0;
       read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
     } catch (org.apache.thrift.TException te) {
       throw new java.io.IOException(te);
@@ -315,6 +394,14 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 2: // CAN_RETRY
+            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
+              struct.canRetry = iprot.readBool();
+              struct.setCanRetryIsSet(true);
+            } else {
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -335,6 +422,9 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
         oprot.writeString(struct.message);
         oprot.writeFieldEnd();
       }
+      oprot.writeFieldBegin(CAN_RETRY_FIELD_DESC);
+      oprot.writeBool(struct.canRetry);
+      oprot.writeFieldEnd();
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -356,20 +446,30 @@ public class IOError extends org.apache.thrift.TException implements org.apache.
       if (struct.isSetMessage()) {
         optionals.set(0);
       }
-      oprot.writeBitSet(optionals, 1);
+      if (struct.isSetCanRetry()) {
+        optionals.set(1);
+      }
+      oprot.writeBitSet(optionals, 2);
       if (struct.isSetMessage()) {
         oprot.writeString(struct.message);
       }
+      if (struct.isSetCanRetry()) {
+        oprot.writeBool(struct.canRetry);
+      }
     }
 
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, IOError struct) throws org.apache.thrift.TException {
       org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
-      java.util.BitSet incoming = iprot.readBitSet(1);
+      java.util.BitSet incoming = iprot.readBitSet(2);
       if (incoming.get(0)) {
         struct.message = iprot.readString();
         struct.setMessageIsSet(true);
       }
+      if (incoming.get(1)) {
+        struct.canRetry = iprot.readBool();
+        struct.setCanRetryIsSet(true);
+      }
     }
   }
 

hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java

@@ -186,6 +186,7 @@ public class ThriftHBaseServiceHandler extends HBaseServiceHandler implements TH
 
   private TIOError getTIOError(IOException e) {
     TIOError err = new TIOErrorWithCause(e);
+    err.setCanRetry(!(e instanceof DoNotRetryIOException));
     err.setMessage(e.getMessage());
     return err;
   }

hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java

@@ -17,15 +17,17 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIOError");
 
   private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
+  private static final org.apache.thrift.protocol.TField CAN_RETRY_FIELD_DESC = new org.apache.thrift.protocol.TField("canRetry", org.apache.thrift.protocol.TType.BOOL, (short) 2);
 
   private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new TIOErrorStandardSchemeFactory();
   private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new TIOErrorTupleSchemeFactory();
 
   public @org.apache.thrift.annotation.Nullable java.lang.String message; // optional
+  public boolean canRetry; // optional
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    MESSAGE((short)1, "message");
+    MESSAGE((short) 1, "message"), CAN_RETRY((short) 2, "canRetry");
 
     private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
 
@@ -43,6 +45,8 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
       switch(fieldId) {
         case 1: // MESSAGE
           return MESSAGE;
+        case 2: // CAN_RETRY
+          return CAN_RETRY;
         default:
           return null;
       }
@@ -84,12 +88,16 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
   }
 
   // isset id assignments
-  private static final _Fields optionals[] = {_Fields.MESSAGE};
+  private static final int __CANRETRY_ISSET_ID = 0;
+  private byte __isset_bitfield = 0;
+  private static final _Fields optionals[] = { _Fields.MESSAGE, _Fields.CAN_RETRY };
   public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
   static {
     java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
     tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.OPTIONAL,
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+    tmpMap.put(_Fields.CAN_RETRY, new org.apache.thrift.meta_data.FieldMetaData("canRetry", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
     metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TIOError.class, metaDataMap);
   }
@@ -101,9 +109,11 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
    * Performs a deep copy on <i>other</i>.
    */
   public TIOError(TIOError other) {
+    __isset_bitfield = other.__isset_bitfield;
     if (other.isSetMessage()) {
       this.message = other.message;
     }
+    this.canRetry = other.canRetry;
   }
 
   public TIOError deepCopy() {
@@ -113,6 +123,8 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
   @Override
   public void clear() {
     this.message = null;
+    setCanRetryIsSet(false);
+    this.canRetry = false;
   }
 
   @org.apache.thrift.annotation.Nullable
@@ -140,6 +152,29 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
     }
   }
 
+  public boolean isCanRetry() {
+    return this.canRetry;
+  }
+
+  public TIOError setCanRetry(boolean canRetry) {
+    this.canRetry = canRetry;
+    setCanRetryIsSet(true);
+    return this;
+  }
+
+  public void unsetCanRetry() {
+    __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __CANRETRY_ISSET_ID);
+  }
+
+  /** Returns true if field canRetry is set (has been assigned a value) and false otherwise */
+  public boolean isSetCanRetry() {
+    return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __CANRETRY_ISSET_ID);
+  }
+
+  public void setCanRetryIsSet(boolean value) {
+    __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __CANRETRY_ISSET_ID, value);
+  }
+
   public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
     switch (field) {
       case MESSAGE:
@@ -150,6 +185,14 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
         }
         break;
 
+      case CAN_RETRY:
+        if (value == null) {
+          unsetCanRetry();
+        } else {
+          setCanRetry((java.lang.Boolean) value);
+        }
+        break;
+
     }
   }
 
@@ -159,6 +202,9 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
       case MESSAGE:
         return getMessage();
 
+      case CAN_RETRY:
+        return isCanRetry();
+
     }
     throw new java.lang.IllegalStateException();
   }
@@ -172,30 +218,34 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
     switch (field) {
       case MESSAGE:
         return isSetMessage();
+      case CAN_RETRY:
+        return isSetCanRetry();
     }
     throw new java.lang.IllegalStateException();
   }
 
   @Override
   public boolean equals(java.lang.Object that) {
-    if (that instanceof TIOError)
+    if (that instanceof TIOError) return this.equals((TIOError) that);
-      return this.equals((TIOError)that);
     return false;
   }
 
   public boolean equals(TIOError that) {
-    if (that == null)
+    if (that == null) return false;
-      return false;
+    if (this == that) return true;
-    if (this == that)
-      return true;
 
     boolean this_present_message = true && this.isSetMessage();
     boolean that_present_message = true && that.isSetMessage();
     if (this_present_message || that_present_message) {
-      if (!(this_present_message && that_present_message))
+      if (!(this_present_message && that_present_message)) return false;
-        return false;
+      if (!this.message.equals(that.message)) return false;
-      if (!this.message.equals(that.message))
+    }
-        return false;
+
+    boolean this_present_canRetry = true && this.isSetCanRetry();
+    boolean that_present_canRetry = true && that.isSetCanRetry();
+    if (this_present_canRetry || that_present_canRetry) {
+      if (!(this_present_canRetry && that_present_canRetry)) return false;
+      if (this.canRetry != that.canRetry) return false;
     }
 
     return true;
@@ -206,8 +256,10 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
     int hashCode = 1;
 
     hashCode = hashCode * 8191 + ((isSetMessage()) ? 131071 : 524287);
-    if (isSetMessage())
+    if (isSetMessage()) hashCode = hashCode * 8191 + message.hashCode();
-      hashCode = hashCode * 8191 + message.hashCode();
+
+    hashCode = hashCode * 8191 + ((isSetCanRetry()) ? 131071 : 524287);
+    if (isSetCanRetry()) hashCode = hashCode * 8191 + ((canRetry) ? 131071 : 524287);
 
     return hashCode;
   }
@@ -230,6 +282,16 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
         return lastComparison;
       }
     }
+    lastComparison = java.lang.Boolean.compare(isSetCanRetry(), other.isSetCanRetry());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetCanRetry()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.canRetry, other.canRetry);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -260,6 +322,12 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
       }
       first = false;
     }
+    if (isSetCanRetry()) {
+      if (!first) sb.append(", ");
+      sb.append("canRetry:");
+      sb.append(this.canRetry);
+      first = false;
+    }
     sb.append(")");
     return sb.toString();
   }
@@ -279,6 +347,8 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
 
   private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException {
     try {
+      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+      __isset_bitfield = 0;
       read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
     } catch (org.apache.thrift.TException te) {
       throw new java.io.IOException(te);
@@ -296,8 +366,7 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
     public void read(org.apache.thrift.protocol.TProtocol iprot, TIOError struct) throws org.apache.thrift.TException {
       org.apache.thrift.protocol.TField schemeField;
       iprot.readStructBegin();
-      while (true)
+      while (true) {
-      {
         schemeField = iprot.readFieldBegin();
         if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
           break;
@@ -311,6 +380,14 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 2: // CAN_RETRY
+            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
+              struct.canRetry = iprot.readBool();
+              struct.setCanRetryIsSet(true);
+            } else {
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -333,6 +410,11 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
           oprot.writeFieldEnd();
         }
       }
+      if (struct.isSetCanRetry()) {
+        oprot.writeFieldBegin(CAN_RETRY_FIELD_DESC);
+        oprot.writeBool(struct.canRetry);
+        oprot.writeFieldEnd();
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -348,31 +430,44 @@ public class TIOError extends org.apache.thrift.TException implements org.apache
   private static class TIOErrorTupleScheme extends org.apache.thrift.scheme.TupleScheme<TIOError> {
 
     @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TIOError struct) throws org.apache.thrift.TException {
+    public void write(org.apache.thrift.protocol.TProtocol prot, TIOError struct)
+        throws org.apache.thrift.TException {
       org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
       java.util.BitSet optionals = new java.util.BitSet();
       if (struct.isSetMessage()) {
         optionals.set(0);
       }
-      oprot.writeBitSet(optionals, 1);
+      if (struct.isSetCanRetry()) {
+        optionals.set(1);
+      }
+      oprot.writeBitSet(optionals, 2);
       if (struct.isSetMessage()) {
         oprot.writeString(struct.message);
       }
+      if (struct.isSetCanRetry()) {
+        oprot.writeBool(struct.canRetry);
+      }
     }
 
     @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TIOError struct) throws org.apache.thrift.TException {
+    public void read(org.apache.thrift.protocol.TProtocol prot, TIOError struct)
+      throws org.apache.thrift.TException {
       org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
-      java.util.BitSet incoming = iprot.readBitSet(1);
+      java.util.BitSet incoming = iprot.readBitSet(2);
       if (incoming.get(0)) {
         struct.message = iprot.readString();
         struct.setMessageIsSet(true);
       }
+      if (incoming.get(1)) {
+        struct.canRetry = iprot.readBool();
+        struct.setCanRetryIsSet(true);
+      }
     }
   }
 
   private static <S extends org.apache.thrift.scheme.IScheme> S scheme(org.apache.thrift.protocol.TProtocol proto) {
-    return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme();
+    return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ?
+      STANDARD_SCHEME_FACTORY :
+      TUPLE_SCHEME_FACTORY).getScheme();
   }
 }
-

hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift

@@ -174,6 +174,7 @@ struct TAppend {
  */
 exception IOError {
   1:string message
+  2:bool canRetry
 }
 
 /**

hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift

@@ -511,6 +511,7 @@ struct TOnlineLogRecord {
  */
 exception TIOError {
   1: optional string message
+  2: optional bool canRetry
 }
 
 /**