From d7d3beb6bca38f0bc4d715963863e32114b762c4 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Wed, 17 Oct 2018 18:04:24 +0800 Subject: [PATCH] HBASE-21330 ReopenTableRegionsProcedure will enter an infinite loop if we schedule a TRSP at the same time --- .../hadoop/hbase/procedure2/Procedure.java | 3 +- .../ReopenTableRegionsProcedure.java | 27 ++++-- ...openTableRegionsProcedureInfiniteLoop.java | 90 +++++++++++++++++++ 3 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureInfiniteLoop.java diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java index 08c7ce3babd..0824b51d7f5 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java @@ -322,7 +322,8 @@ public abstract class Procedure implements Comparable l.getSeqNum() >= 0)) { + if (regions.stream().anyMatch(loc -> canSchedule(env, loc))) { attempt = 0; setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_REOPEN_REGIONS); return Flow.HAS_MORE_STATE; } - // All the regions need to reopen are in OPENING state which means we can not schedule any - // MRPs. + // We can not schedule TRSP for all the regions need to reopen, wait for a while and retry + // again. long backoff = ProcedureUtil.getBackoffTimeMs(this.attempt++); LOG.info( "There are still {} region(s) which need to be reopened for table {} are in " + @@ -138,6 +154,7 @@ public class ReopenTableRegionsProcedure env.getProcedureScheduler().addFront(this); return false; // 'false' means that this procedure handled the timeout } + @Override protected void rollbackState(MasterProcedureEnv env, ReopenTableRegionsState state) throws IOException, InterruptedException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureInfiniteLoop.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureInfiniteLoop.java new file mode 100644 index 00000000000..870f3bf817a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureInfiniteLoop.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import java.io.IOException; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.RegionStateNode; +import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Testcase for HBASE-21330. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestReopenTableRegionsProcedureInfiniteLoop { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestReopenTableRegionsProcedureInfiniteLoop.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static TableName TABLE_NAME = TableName.valueOf("InfiniteLoop"); + + private static byte[] CF = Bytes.toBytes("cf"); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + UTIL.startMiniCluster(1); + UTIL.createTable(TABLE_NAME, CF); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testInfiniteLoop() throws IOException { + HMaster master = UTIL.getMiniHBaseCluster().getMaster(); + AssignmentManager am = master.getAssignmentManager(); + ProcedureExecutor exec = master.getMasterProcedureExecutor(); + RegionInfo regionInfo = UTIL.getAdmin().getRegions(TABLE_NAME).get(0); + RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(regionInfo); + long procId; + ReopenTableRegionsProcedure proc = new ReopenTableRegionsProcedure(TABLE_NAME); + regionNode.lock(); + try { + procId = exec.submitProcedure(proc); + UTIL.waitFor(30000, () -> proc.hasLock()); + TransitRegionStateProcedure trsp = + TransitRegionStateProcedure.reopen(exec.getEnvironment(), regionInfo); + regionNode.setProcedure(trsp); + exec.submitProcedure(trsp); + } finally { + regionNode.unlock(); + } + UTIL.waitFor(60000, () -> exec.isFinished(procId)); + } +}