Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-7302. Upgrading to JUnit 4.13 causes testcase TestFetcher.testCorruptedIFile() to fail. Contributed by Peter Bacsko. Reviewed by Akira Ajisaka. 

(cherry picked from commit da93cd962c)

Conflicts:
	hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java

(cherry picked from commit 7ad844ee9e)
This commit is contained in:
Eric Badger 2021-04-23 19:01:29 +00:00
parent 63b43a2201
commit be5fa9615d
2 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
View File

@ -29,6 +29,7 @@ import org.apache.hadoop.io.nativeio.NativeIO;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_WILLNEED; import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_WILLNEED;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -60,6 +61,16 @@ public class ReadaheadPool {
} }
} }
@VisibleForTesting
public static void resetInstance() {
synchronized (ReadaheadPool.class) {
if (instance != null) {
instance.pool.shutdownNow();
instance = null;
}
}
}
private ReadaheadPool() { private ReadaheadPool() {
pool = new ThreadPoolExecutor(POOL_SIZE, MAX_POOL_SIZE, 3L, TimeUnit.SECONDS, pool = new ThreadPoolExecutor(POOL_SIZE, MAX_POOL_SIZE, 3L, TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(CAPACITY)); new ArrayBlockingQueue<Runnable>(CAPACITY));

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
View File

@ -47,6 +47,7 @@ import java.util.ArrayList;
import javax.crypto.SecretKey; import javax.crypto.SecretKey;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.IFileInputStream; import org.apache.hadoop.mapred.IFileInputStream;
@ -93,6 +94,8 @@ public class TestFetcher {
@SuppressWarnings("unchecked") // mocked generics @SuppressWarnings("unchecked") // mocked generics
public void setup() { public void setup() {
LOG.info(">>>> " + name.getMethodName()); LOG.info(">>>> " + name.getMethodName());
// to avoid threading issues with JUnit 4.13+
ReadaheadPool.resetInstance();
job = new JobConf(); job = new JobConf();
job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false); job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
jobWithRetry = new JobConf(); jobWithRetry = new JobConf();