diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java index 5e9e97e219d..d39013bd270 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java @@ -22,6 +22,8 @@ import io.netty.util.Timeout; import io.netty.util.TimerTask; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.UnknownHostException; import java.util.concurrent.TimeUnit; @@ -45,6 +47,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.TokenSelector; +import org.apache.hadoop.util.Time; /** * Base class for ipc connection. @@ -75,6 +78,14 @@ abstract class RpcConnection { // the last time we were picked up from connection pool. protected long lastTouched; + // Determines whether we need to do an explicit clearing of kerberos tickets with relogin + private boolean forceReloginEnabled; + // Minimum time between two force re-login attempts + private int minTimeBeforeForceRelogin; + + // Time when last forceful re-login was attempted + private long lastForceReloginAttempt = -1; + protected RpcConnection(Configuration conf, HashedWheelTimer timeoutTimer, ConnectionId remoteId, String clusterId, boolean isSecurityEnabled, Codec codec, CompressionCodec compressor) throws IOException { @@ -126,8 +137,14 @@ abstract class RpcConnection { LOG.debug("Use " + authMethod + " authentication for service " + remoteId.serviceName + ", sasl=" + useSasl); } - reloginMaxBackoff = conf.getInt("hbase.security.relogin.maxbackoff", 5000); + + reloginMaxBackoff = conf.getInt(HConstants.HBASE_RELOGIN_MAXBACKOFF, 5000); this.remoteId = remoteId; + + forceReloginEnabled = conf.getBoolean(HConstants.HBASE_FORCE_RELOGIN_ENABLED, true); + // Default minimum time between force relogin attempts is 10 minutes + this.minTimeBeforeForceRelogin = + conf.getInt(HConstants.HBASE_MINTIME_BEFORE_FORCE_RELOGIN, 10 * 60 * 1000); } private UserInformation getUserInfo(UserGroupInformation ugi) { @@ -173,12 +190,62 @@ abstract class RpcConnection { protected void relogin() throws IOException { if (UserGroupInformation.isLoginKeytabBased()) { - UserGroupInformation.getLoginUser().reloginFromKeytab(); + if (shouldForceRelogin()) { + LOG.debug( + "SASL Authentication failure. Attempting a forceful re-login for " + + UserGroupInformation.getLoginUser().getUserName()); + Method logoutUserFromKeytab; + Method forceReloginFromKeytab; + try { + logoutUserFromKeytab = UserGroupInformation.class.getMethod("logoutUserFromKeytab"); + forceReloginFromKeytab = UserGroupInformation.class.getMethod("forceReloginFromKeytab"); + } catch (NoSuchMethodException e) { + // This shouldn't happen as we already check for the existence of these methods before + // entering this block + throw new RuntimeException("Cannot find forceReloginFromKeytab method in UGI"); + } + logoutUserFromKeytab.setAccessible(true); + forceReloginFromKeytab.setAccessible(true); + try { + logoutUserFromKeytab.invoke(UserGroupInformation.getLoginUser()); + forceReloginFromKeytab.invoke(UserGroupInformation.getLoginUser()); + } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + throw new RuntimeException(e.getCause()); + } + } else { + UserGroupInformation.getLoginUser().reloginFromKeytab(); + } } else { UserGroupInformation.getLoginUser().reloginFromTicketCache(); } } + private boolean shouldForceRelogin() { + if (!forceReloginEnabled) { + return false; + } + long now = Time.now(); + // If the last force relogin attempted is less than the configured minimum time, revert to the + // default relogin method of UGI + if (lastForceReloginAttempt != -1 + && (now - lastForceReloginAttempt < minTimeBeforeForceRelogin)) { + LOG.debug("Not attempting to force re-login since the last attempt is less than " + + minTimeBeforeForceRelogin + " millis"); + return false; + } + try { + // Check if forceRelogin method is available in UGI using reflection + UserGroupInformation.class.getMethod("forceReloginFromKeytab"); + UserGroupInformation.class.getMethod("logoutUserFromKeytab"); + } catch (NoSuchMethodException e) { + LOG.debug( + "forceReloginFromKeytab method not available in UGI. Skipping to attempt force relogin"); + return false; + } + lastForceReloginAttempt = now; + return true; + } + protected void scheduleTimeoutTask(final Call call) { if (call.timeout > 0) { call.timeoutTask = timeoutTimer.newTimeout(new TimerTask() { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index e660fe32671..c1d688e74eb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -321,6 +321,19 @@ public final class HConstants { public static final String HBASE_CLIENT_META_OPERATION_TIMEOUT = "hbase.client.meta.operation.timeout"; + /** Parameter name for HBase client max backoff across SASL relogin failure retries */ + public static final String HBASE_RELOGIN_MAXBACKOFF = "hbase.security.relogin.maxbackoff"; + + /** Parameter name for HBase client minimum time between forceful relogin attempts */ + public static final String HBASE_MINTIME_BEFORE_FORCE_RELOGIN = + "hbase.mintime.before.force.relogin"; + + /** + * Whether forceful relogin (explicit clearing of kerberos tickets) is enabled on SASL + * Authentication failure + */ + public static final String HBASE_FORCE_RELOGIN_ENABLED = "hbase.security.force.relogin.enabled"; + /** Default HBase client operation timeout, which is tantamount to a blocking call */ public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = 1200000; @@ -331,6 +344,7 @@ public final class HConstants { /** Default HBase client meta replica scan call timeout, 1 second */ public static final int HBASE_CLIENT_META_REPLICA_SCAN_TIMEOUT_DEFAULT = 1000000; + /** Used to construct the name of the log directory for a region server */ public static final String HREGION_LOGDIR_NAME = "WALs";