Signed-off-by: Xin Sun <ddupgs@gmail.com>
(cherry picked from commit 41972cb460
)
This commit is contained in:
parent
92525fb869
commit
6dd1b58481
|
@ -144,6 +144,8 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
|
||||||
import org.apache.hadoop.hdfs.DFSClient;
|
import org.apache.hadoop.hdfs.DFSClient;
|
||||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
|
import org.apache.hadoop.hdfs.server.datanode.DataNode;
|
||||||
|
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
|
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
|
||||||
import org.apache.hadoop.mapred.JobConf;
|
import org.apache.hadoop.mapred.JobConf;
|
||||||
import org.apache.hadoop.mapred.MiniMRCluster;
|
import org.apache.hadoop.mapred.MiniMRCluster;
|
||||||
|
@ -196,6 +198,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
||||||
public static final boolean PRESPLIT_TEST_TABLE = true;
|
public static final boolean PRESPLIT_TEST_TABLE = true;
|
||||||
|
|
||||||
private MiniDFSCluster dfsCluster = null;
|
private MiniDFSCluster dfsCluster = null;
|
||||||
|
private FsDatasetAsyncDiskServiceFixer dfsClusterFixer = null;
|
||||||
|
|
||||||
private volatile HBaseCluster hbaseCluster = null;
|
private volatile HBaseCluster hbaseCluster = null;
|
||||||
private MiniMRCluster mrCluster = null;
|
private MiniMRCluster mrCluster = null;
|
||||||
|
@ -574,6 +577,56 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
||||||
return getTestFileSystem().delete(cpath, true);
|
return getTestFileSystem().delete(cpath, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Workaround to avoid IllegalThreadStateException
|
||||||
|
// See HBASE-27148 for more details
|
||||||
|
private static final class FsDatasetAsyncDiskServiceFixer extends Thread {
|
||||||
|
|
||||||
|
private volatile boolean stopped = false;
|
||||||
|
|
||||||
|
private final MiniDFSCluster cluster;
|
||||||
|
|
||||||
|
FsDatasetAsyncDiskServiceFixer(MiniDFSCluster cluster) {
|
||||||
|
super("FsDatasetAsyncDiskServiceFixer");
|
||||||
|
setDaemon(true);
|
||||||
|
this.cluster = cluster;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
while (!stopped) {
|
||||||
|
try {
|
||||||
|
Thread.sleep(30000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// we could add new datanodes during tests, so here we will check every 30 seconds, as the
|
||||||
|
// timeout of the thread pool executor is 60 seconds by default.
|
||||||
|
try {
|
||||||
|
for (DataNode dn : cluster.getDataNodes()) {
|
||||||
|
FsDatasetSpi<?> dataset = dn.getFSDataset();
|
||||||
|
Field service = dataset.getClass().getDeclaredField("asyncDiskService");
|
||||||
|
service.setAccessible(true);
|
||||||
|
Object asyncDiskService = service.get(dataset);
|
||||||
|
Field group = asyncDiskService.getClass().getDeclaredField("threadGroup");
|
||||||
|
group.setAccessible(true);
|
||||||
|
ThreadGroup threadGroup = (ThreadGroup) group.get(asyncDiskService);
|
||||||
|
if (threadGroup.isDaemon()) {
|
||||||
|
threadGroup.setDaemon(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.warn("failed to reset thread pool timeout for FsDatasetAsyncDiskService", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void shutdown() {
|
||||||
|
stopped = true;
|
||||||
|
interrupt();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start a minidfscluster.
|
* Start a minidfscluster.
|
||||||
* @param servers How many DNs to start. n * @see #shutdownMiniDFSCluster()
|
* @param servers How many DNs to start. n * @see #shutdownMiniDFSCluster()
|
||||||
|
@ -632,7 +685,8 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
||||||
|
|
||||||
this.dfsCluster =
|
this.dfsCluster =
|
||||||
new MiniDFSCluster(0, this.conf, servers, true, true, true, null, racks, hosts, null);
|
new MiniDFSCluster(0, this.conf, servers, true, true, true, null, racks, hosts, null);
|
||||||
|
this.dfsClusterFixer = new FsDatasetAsyncDiskServiceFixer(dfsCluster);
|
||||||
|
this.dfsClusterFixer.start();
|
||||||
// Set this just-started cluster as our filesystem.
|
// Set this just-started cluster as our filesystem.
|
||||||
setFs();
|
setFs();
|
||||||
|
|
||||||
|
@ -656,6 +710,8 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
||||||
"ERROR");
|
"ERROR");
|
||||||
dfsCluster =
|
dfsCluster =
|
||||||
new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
|
new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
|
||||||
|
this.dfsClusterFixer = new FsDatasetAsyncDiskServiceFixer(dfsCluster);
|
||||||
|
this.dfsClusterFixer.start();
|
||||||
return dfsCluster;
|
return dfsCluster;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -778,6 +834,12 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
||||||
// The below throws an exception per dn, AsynchronousCloseException.
|
// The below throws an exception per dn, AsynchronousCloseException.
|
||||||
this.dfsCluster.shutdown();
|
this.dfsCluster.shutdown();
|
||||||
dfsCluster = null;
|
dfsCluster = null;
|
||||||
|
// It is possible that the dfs cluster is set through setDFSCluster method, where we will not
|
||||||
|
// have a fixer
|
||||||
|
if (dfsClusterFixer != null) {
|
||||||
|
this.dfsClusterFixer.shutdown();
|
||||||
|
dfsClusterFixer = null;
|
||||||
|
}
|
||||||
dataTestDirOnTestFS = null;
|
dataTestDirOnTestFS = null;
|
||||||
CommonFSUtils.setFsDefault(this.conf, new Path("file:///"));
|
CommonFSUtils.setFsDefault(this.conf, new Path("file:///"));
|
||||||
}
|
}
|
||||||
|
|
|
@ -128,6 +128,16 @@
|
||||||
<type>test-jar</type>
|
<type>test-jar</type>
|
||||||
<scope>compile</scope>
|
<scope>compile</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mockito</groupId>
|
||||||
|
<artifactId>mockito-core</artifactId>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.github.stephenc.findbugs</groupId>
|
||||||
|
<artifactId>findbugs-annotations</artifactId>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.slf4j</groupId>
|
<groupId>org.slf4j</groupId>
|
||||||
<artifactId>jcl-over-slf4j</artifactId>
|
<artifactId>jcl-over-slf4j</artifactId>
|
||||||
|
|
121
pom.xml
121
pom.xml
|
@ -544,7 +544,7 @@
|
||||||
<java.min.version>${compileSource}</java.min.version>
|
<java.min.version>${compileSource}</java.min.version>
|
||||||
<!-- Dependencies -->
|
<!-- Dependencies -->
|
||||||
<hadoop-two.version>2.10.0</hadoop-two.version>
|
<hadoop-two.version>2.10.0</hadoop-two.version>
|
||||||
<hadoop-three.version>3.1.2</hadoop-three.version>
|
<hadoop-three.version>3.2.3</hadoop-three.version>
|
||||||
<!-- These must be defined here for downstream build tools that don't look at profiles.
|
<!-- These must be defined here for downstream build tools that don't look at profiles.
|
||||||
They ought to match the values found in our default hadoop profile, which is
|
They ought to match the values found in our default hadoop profile, which is
|
||||||
currently "hadoop-2.0". See HBASE-15925 for more info. -->
|
currently "hadoop-2.0". See HBASE-15925 for more info. -->
|
||||||
|
@ -557,7 +557,7 @@
|
||||||
<netty.hadoop.version>3.6.2.Final</netty.hadoop.version>
|
<netty.hadoop.version>3.6.2.Final</netty.hadoop.version>
|
||||||
<!-- end HBASE-15925 default hadoop compatibility values -->
|
<!-- end HBASE-15925 default hadoop compatibility values -->
|
||||||
<audience-annotations.version>0.5.0</audience-annotations.version>
|
<audience-annotations.version>0.5.0</audience-annotations.version>
|
||||||
<avro.version>1.7.7</avro.version>
|
<avro.version>1.11.0</avro.version>
|
||||||
<caffeine.version>2.8.1</caffeine.version>
|
<caffeine.version>2.8.1</caffeine.version>
|
||||||
<commons-codec.version>1.13</commons-codec.version>
|
<commons-codec.version>1.13</commons-codec.version>
|
||||||
<commons-io.version>2.11.0</commons-io.version>
|
<commons-io.version>2.11.0</commons-io.version>
|
||||||
|
@ -2816,8 +2816,6 @@
|
||||||
--add-opens java.base/java.lang.reflect=ALL-UNNAMED
|
--add-opens java.base/java.lang.reflect=ALL-UNNAMED
|
||||||
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
|
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
|
||||||
${hbase-surefire.argLine}</argLine>
|
${hbase-surefire.argLine}</argLine>
|
||||||
<!-- We need a minimum HDFS version of 3.2.0 for HADOOP-12760 -->
|
|
||||||
<hadoop-three.version>3.2.0</hadoop-three.version>
|
|
||||||
<!--
|
<!--
|
||||||
Value to use for surefire when running jdk11.
|
Value to use for surefire when running jdk11.
|
||||||
TODO: replicate logic for windows
|
TODO: replicate logic for windows
|
||||||
|
@ -3643,12 +3641,16 @@
|
||||||
<artifactId>jersey-core</artifactId>
|
<artifactId>jersey-core</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
<groupId>javax.xml.bind</groupId>
|
||||||
<artifactId>jackson-jaxrs</artifactId>
|
<artifactId>jaxb-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.ws.rs</groupId>
|
||||||
|
<artifactId>jsr311-api</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
<artifactId>jackson-xc</artifactId>
|
<artifactId>*</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>io.netty</groupId>
|
<groupId>io.netty</groupId>
|
||||||
|
@ -3666,14 +3668,6 @@
|
||||||
<groupId>javax.inject</groupId>
|
<groupId>javax.inject</groupId>
|
||||||
<artifactId>javax.inject</artifactId>
|
<artifactId>javax.inject</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
|
||||||
<artifactId>jackson-core-asl</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
|
||||||
<artifactId>jackson-mapper-asl</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>com.google.guava</groupId>
|
<groupId>com.google.guava</groupId>
|
||||||
<artifactId>guava</artifactId>
|
<artifactId>guava</artifactId>
|
||||||
|
@ -3702,12 +3696,8 @@
|
||||||
<version>${hadoop-three.version}</version>
|
<version>${hadoop-three.version}</version>
|
||||||
<exclusions>
|
<exclusions>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>com.sun.jersey</groupId>
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
<artifactId>jersey-core</artifactId>
|
<artifactId>*</artifactId>
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>io.netty</groupId>
|
|
||||||
<artifactId>netty</artifactId>
|
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<!--HERE-->
|
<!--HERE-->
|
||||||
<exclusion>
|
<exclusion>
|
||||||
|
@ -3777,19 +3767,7 @@
|
||||||
<exclusions>
|
<exclusions>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
<artifactId>jackson-mapper-asl</artifactId>
|
<artifactId>*</artifactId>
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
|
||||||
<artifactId>jackson-core-asl</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
|
||||||
<artifactId>jackson-jaxrs</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
|
||||||
<artifactId>jackson-xc</artifactId>
|
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>javax.xml.bind</groupId>
|
<groupId>javax.xml.bind</groupId>
|
||||||
|
@ -3860,11 +3838,7 @@
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
<artifactId>jackson-core-asl</artifactId>
|
<artifactId>*</artifactId>
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
|
||||||
<artifactId>jackson-mapper-asl</artifactId>
|
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>com.google.guava</groupId>
|
<groupId>com.google.guava</groupId>
|
||||||
|
@ -3929,11 +3903,7 @@
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
<artifactId>jackson-core-asl</artifactId>
|
<artifactId>*</artifactId>
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
|
||||||
<artifactId>jackson-mapper-asl</artifactId>
|
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>com.google.guava</groupId>
|
<groupId>com.google.guava</groupId>
|
||||||
|
@ -3968,38 +3938,6 @@
|
||||||
-->
|
-->
|
||||||
</exclusions>
|
</exclusions>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<!-- Is this needed? Seems a duplicate of the above dependency but for the
|
|
||||||
classifier-->
|
|
||||||
<groupId>org.apache.hadoop</groupId>
|
|
||||||
<artifactId>hadoop-hdfs</artifactId>
|
|
||||||
<version>${hadoop-three.version}</version>
|
|
||||||
<classifier>tests</classifier>
|
|
||||||
<type>test-jar</type>
|
|
||||||
<scope>test</scope>
|
|
||||||
<exclusions>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>com.sun.jersey</groupId>
|
|
||||||
<artifactId>jersey-core</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.slf4j</groupId>
|
|
||||||
<artifactId>slf4j-log4j12</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>log4j</groupId>
|
|
||||||
<artifactId>log4j</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>ch.qos.reload4j</groupId>
|
|
||||||
<artifactId>reload4j</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.slf4j</groupId>
|
|
||||||
<artifactId>slf4j-reload4j</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
</exclusions>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.hadoop</groupId>
|
<groupId>org.apache.hadoop</groupId>
|
||||||
<artifactId>hadoop-auth</artifactId>
|
<artifactId>hadoop-auth</artifactId>
|
||||||
|
@ -4081,12 +4019,8 @@
|
||||||
<artifactId>junit</artifactId>
|
<artifactId>junit</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.codehause.jackson</groupId>
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
<artifactId>jackson-core-asl</artifactId>
|
<artifactId>*</artifactId>
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.codehause.jackson</groupId>
|
|
||||||
<artifactId>jackson-mapper-asl</artifactId>
|
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.slf4j</groupId>
|
<groupId>org.slf4j</groupId>
|
||||||
|
@ -4182,6 +4116,29 @@
|
||||||
<groupId>org.slf4j</groupId>
|
<groupId>org.slf4j</groupId>
|
||||||
<artifactId>slf4j-reload4j</artifactId>
|
<artifactId>slf4j-reload4j</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
|
<!--
|
||||||
|
Needed in test context when hadoop-3.3 runs.
|
||||||
|
<exclusion>
|
||||||
|
<groupId>io.netty</groupId>
|
||||||
|
<artifactId>netty-all</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
-->
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
|
<artifactId>*</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.servlet.jsp</groupId>
|
||||||
|
<artifactId>jsp-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.xml.bind</groupId>
|
||||||
|
<artifactId>jaxb-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.ws.rs</groupId>
|
||||||
|
<artifactId>jsr311-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
</exclusions>
|
</exclusions>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
|
|
Loading…
Reference in New Issue