HBASE-14605 Split fails due to 'No valid credentials' error when SecureBulkLoadEndpoint#start tries to access hdfs

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -59,7 +58,7 @@ class SplitRequest implements Runnable {
     return "regionName=" + parent + ", midKey=" + Bytes.toStringBinary(midKey);
   }
 
-  private void doSplitting() {
+  private void doSplitting(User user) {
     boolean success = false;
     server.metricsRegionServer.incrSplitRequest();
     long startTime = EnvironmentEdgeManager.currentTime();
@@ -80,7 +79,7 @@ class SplitRequest implements Runnable {
       // the prepare call -- we are not ready to split just now. Just return.
       if (!st.prepare()) return;
       try {
-        st.execute(this.server, this.server);
+        st.execute(this.server, this.server, user);
         success = true;
       } catch (Exception e) {
         if (this.server.isStopping() || this.server.isStopped()) {
@@ -153,22 +152,7 @@ class SplitRequest implements Runnable {
         this.server.isStopping() + " or stopped=" + this.server.isStopped());
       return;
     }
-    if (this.user == null) doSplitting();
+    doSplitting(user);
-    else {
-      try {
-        user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
-          @Override
-          public Void run() throws Exception {
-            doSplitting();
-            return null;
-          }
-        });
-      } catch (InterruptedException ie) {
-        Thread.currentThread().interrupt();
-      } catch (IOException ioe) {
-        LOG.error("Encountered exception while splitting", ioe);
-      }
-    }
   }
 
   protected void releaseTableLock() {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.PairOfSameType;
 
 /**
@@ -191,9 +192,25 @@ public interface SplitTransaction {
    * @return Regions created
    * @throws IOException
    * @see #rollback(Server, RegionServerServices)
+   * @deprecated use #execute(Server, RegionServerServices, User)
    */
+  @Deprecated
   PairOfSameType<Region> execute(Server server, RegionServerServices services) throws IOException;
 
+  /**
+   * Run the transaction.
+   * @param server Hosting server instance.  Can be null when testing.
+   * @param services Used to online/offline regions.
+   * @param user
+   * @throws IOException If thrown, transaction failed.
+   *          Call {@link #rollback(Server, RegionServerServices)}
+   * @return Regions created
+   * @throws IOException
+   * @see #rollback(Server, RegionServerServices)
+   */
+  PairOfSameType<Region> execute(Server server, RegionServerServices services, User user)
+      throws IOException;
+
   /**
    * Roll back a failed transaction
    * @param server Hosting server instance (May be null when testing).
@@ -201,9 +218,22 @@ public interface SplitTransaction {
    * @throws IOException If thrown, rollback failed.  Take drastic action.
    * @return True if we successfully rolled back, false if we got to the point
    * of no return and so now need to abort the server to minimize damage.
+   * @deprecated use #rollback(Server, RegionServerServices, User)
    */
+  @Deprecated
   boolean rollback(Server server, RegionServerServices services) throws IOException;
 
+  /**
+   * Roll back a failed transaction
+   * @param server Hosting server instance (May be null when testing).
+   * @param services
+   * @param user
+   * @throws IOException If thrown, rollback failed.  Take drastic action.
+   * @return True if we successfully rolled back, false if we got to the point
+   * of no return and so now need to abort the server to minimize damage.
+   */
+  boolean rollback(Server server, RegionServerServices services, User user) throws IOException;
+
   /**
    * Register a listener for transaction preparation, execution, and possibly
    * rollback phases.

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.ListIterator;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -206,13 +208,14 @@ public class SplitTransactionImpl implements SplitTransaction {
    * @param server Hosting server instance.  Can be null when testing (won't try
    * and update in zk if a null server)
    * @param services Used to online/offline regions.
+   * @param user
    * @throws IOException If thrown, transaction failed.
    *    Call {@link #rollback(Server, RegionServerServices)}
    * @return Regions created
    */
   @VisibleForTesting
   PairOfSameType<Region> createDaughters(final Server server,
-      final RegionServerServices services) throws IOException {
+      final RegionServerServices services, User user) throws IOException {
     LOG.info("Starting split of region " + this.parent);
     if ((server != null && server.isStopped()) ||
         (services != null && services.isStopping())) {
@@ -225,9 +228,26 @@ public class SplitTransactionImpl implements SplitTransaction {
 
     // Coprocessor callback
     if (this.parent.getCoprocessorHost() != null) {
-      // TODO: Remove one of these
+      if (user == null) {
-      this.parent.getCoprocessorHost().preSplit();
+        // TODO: Remove one of these
-      this.parent.getCoprocessorHost().preSplit(this.splitrow);
+        parent.getCoprocessorHost().preSplit();
+        parent.getCoprocessorHost().preSplit(splitrow);
+      } else {
+        try {
+          user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
+            @Override
+            public Void run() throws Exception {
+              parent.getCoprocessorHost().preSplit();
+              parent.getCoprocessorHost().preSplit(splitrow);
+              return null;
+            }
+          });
+        } catch (InterruptedException ie) {
+          InterruptedIOException iioe = new InterruptedIOException();
+          iioe.initCause(ie);
+          throw iioe;
+        }
+      }
     }
 
     transition(SplitTransactionPhase.AFTER_PRE_SPLIT_HOOK);
@@ -241,11 +261,27 @@ public class SplitTransactionImpl implements SplitTransaction {
 
     PairOfSameType<Region> daughterRegions = stepsBeforePONR(server, services, testing);
 
-    List<Mutation> metaEntries = new ArrayList<Mutation>();
+    final List<Mutation> metaEntries = new ArrayList<Mutation>();
+    boolean ret = false;
     if (this.parent.getCoprocessorHost() != null) {
-      if (this.parent.getCoprocessorHost().
+      if (user == null) {
-          preSplitBeforePONR(this.splitrow, metaEntries)) {
+        ret = parent.getCoprocessorHost().preSplitBeforePONR(splitrow, metaEntries);
-        throw new IOException("Coprocessor bypassing region "
+      } else {
+        try {
+          ret = user.getUGI().doAs(new PrivilegedExceptionAction<Boolean>() {
+            @Override
+            public Boolean run() throws Exception {
+              return parent.getCoprocessorHost().preSplitBeforePONR(splitrow, metaEntries);
+            }
+          });
+        } catch (InterruptedException ie) {
+          InterruptedIOException iioe = new InterruptedIOException();
+          iioe.initCause(ie);
+          throw iioe;
+        }
+      }
+      if (ret) {
+          throw new IOException("Coprocessor bypassing region "
             + parent.getRegionInfo().getRegionNameAsString() + " split.");
       }
       try {
@@ -449,22 +485,48 @@ public class SplitTransactionImpl implements SplitTransaction {
   }
 
   @Override
-  public PairOfSameType<Region> execute(final Server server, final RegionServerServices services)
+  public PairOfSameType<Region> execute(final Server server,
-      throws IOException {
+    final RegionServerServices services)
+        throws IOException {
+    if (User.isHBaseSecurityEnabled(parent.getBaseConf())) {
+      LOG.warn("Should use execute(Server, RegionServerServices, User)");
+    }
+    return execute(server, services, null);
+  }
+
+  @Override
+  public PairOfSameType<Region> execute(final Server server, final RegionServerServices services,
+    User user) throws IOException {
     this.server = server;
     this.rsServices = services;
-    PairOfSameType<Region> regions = createDaughters(server, services);
+    PairOfSameType<Region> regions = createDaughters(server, services, user);
-    stepsAfterPONR(server, services, regions);
+    stepsAfterPONR(server, services, regions, user);
     transition(SplitTransactionPhase.COMPLETED);
     return regions;
   }
 
   @VisibleForTesting
   void stepsAfterPONR(final Server server,
-      final RegionServerServices services, PairOfSameType<Region> regions)
+      final RegionServerServices services, final PairOfSameType<Region> regions, User user)
       throws IOException {
     if (this.parent.getCoprocessorHost() != null) {
-      this.parent.getCoprocessorHost().preSplitAfterPONR();
+      if (user == null) {
+        parent.getCoprocessorHost().preSplitAfterPONR();
+      } else {
+        try {
+          user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
+            @Override
+            public Void run() throws Exception {
+              parent.getCoprocessorHost().preSplitAfterPONR();
+              return null;
+            }
+          });
+        } catch (InterruptedException ie) {
+          InterruptedIOException iioe = new InterruptedIOException();
+          iioe.initCause(ie);
+          throw iioe;
+        }
+      }
     }
 
     openDaughters(server, services, regions.getFirst(), regions.getSecond());
@@ -473,7 +535,23 @@ public class SplitTransactionImpl implements SplitTransaction {
 
     // Coprocessor callback
     if (parent.getCoprocessorHost() != null) {
-      parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond());
+      if (user == null) {
+        this.parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond());
+      } else {
+        try {
+          user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
+            @Override
+            public Void run() throws Exception {
+              parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond());
+              return null;
+            }
+          });
+        } catch (InterruptedException ie) {
+          InterruptedIOException iioe = new InterruptedIOException();
+          iioe.initCause(ie);
+          throw iioe;
+        }
+      }
     }
 
     transition(SplitTransactionPhase.AFTER_POST_SPLIT_HOOK);
@@ -689,11 +767,36 @@ public class SplitTransactionImpl implements SplitTransaction {
   @Override
   public boolean rollback(final Server server, final RegionServerServices services)
       throws IOException {
+    if (User.isHBaseSecurityEnabled(parent.getBaseConf())) {
+      LOG.warn("Should use rollback(Server, RegionServerServices, User)");
+    }
+    return rollback(server, services, null);
+  }
+
+  @Override
+  public boolean rollback(final Server server, final RegionServerServices services, User user)
+      throws IOException {
     this.server = server;
     this.rsServices = services;
     // Coprocessor callback
     if (this.parent.getCoprocessorHost() != null) {
-      this.parent.getCoprocessorHost().preRollBackSplit();
+      if (user == null) {
+        this.parent.getCoprocessorHost().preRollBackSplit();
+      } else {
+        try {
+          user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
+            @Override
+            public Void run() throws Exception {
+              parent.getCoprocessorHost().preRollBackSplit();
+              return null;
+            }
+          });
+        } catch (InterruptedException ie) {
+          InterruptedIOException iioe = new InterruptedIOException();
+          iioe.initCause(ie);
+          throw iioe;
+        }
+      }
     }
 
     boolean result = true;
@@ -772,7 +875,23 @@ public class SplitTransactionImpl implements SplitTransaction {
     }
     // Coprocessor callback
     if (this.parent.getCoprocessorHost() != null) {
-      this.parent.getCoprocessorHost().postRollBackSplit();
+      if (user == null) {
+        this.parent.getCoprocessorHost().postRollBackSplit();
+      } else {
+        try {
+          user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
+            @Override
+            public Void run() throws Exception {
+              parent.getCoprocessorHost().postRollBackSplit();
+              return null;
+            }
+          });
+        } catch (InterruptedException ie) {
+          InterruptedIOException iioe = new InterruptedIOException();
+          iioe.initCause(ie);
+          throw iioe;
+        }
+      }
     }
     return result;
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java

@@ -111,7 +111,7 @@ public class TestEndToEndSplitTransaction {
       split.prepare();
 
       // 1. phase I
-      PairOfSameType<Region> regions = split.createDaughters(server, server);
+      PairOfSameType<Region> regions = split.createDaughters(server, server, null);
       assertFalse(test(conn, tableName, firstRow, server));
       assertFalse(test(conn, tableName, lastRow, server));
 

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java

@@ -88,6 +88,7 @@ import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.Repor
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -652,7 +653,7 @@ public class TestSplitTransactionOnCluster {
       SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
       try {
         st.prepare();
-        st.createDaughters(regionServer, regionServer);
+        st.createDaughters(regionServer, regionServer, null);
       } catch (IOException e) {
 
       }
@@ -1374,7 +1375,7 @@ public class TestSplitTransactionOnCluster {
         throws IOException {
       RegionCoprocessorEnvironment environment = ctx.getEnvironment();
       HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
-      st.stepsAfterPONR(rs, rs, daughterRegions);
+      st.stepsAfterPONR(rs, rs, daughterRegions, null);
     }
   }