Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-6881. Fix warnings from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.

This commit is contained in:
Akira Ajisaka 2017-04-27 15:45:33 +09:00
parent 28eb2aabeb
commit 3ed3062fe3
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
6 changed files with 53 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
View File

@ -27,6 +27,8 @@ import java.lang.management.ThreadMXBean;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.Collections;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@ -81,7 +83,7 @@ public class LocalContainerLauncher extends AbstractService implements
private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
private FileContext curFC = null;
private final HashSet<File> localizedFiles;
private Set<File> localizedFiles = new HashSet<File>();
private final AppContext context;
private final TaskUmbilicalProtocol umbilical;
private final ClassLoader jobClassLoader;
@ -121,9 +123,12 @@ public class LocalContainerLauncher extends AbstractService implements
// users who do that get what they deserve (and will have to disable
// uberization in order to run correctly).
File[] curLocalFiles = curDir.listFiles();
localizedFiles = new HashSet<File>(curLocalFiles.length);
for (int j = 0; j < curLocalFiles.length; ++j) {
localizedFiles.add(curLocalFiles[j]);
if (curLocalFiles != null) {
HashSet<File> lf = new HashSet<File>(curLocalFiles.length);
for (int j = 0; j < curLocalFiles.length; ++j) {
lf.add(curLocalFiles[j]);
}
localizedFiles = Collections.unmodifiableSet(lf);
}
// Relocalization note/future FIXME (per chrisdo, 20110315): At moment,
@ -521,26 +526,29 @@ public class LocalContainerLauncher extends AbstractService implements
*/
private void relocalize() {
File[] curLocalFiles = curDir.listFiles();
for (int j = 0; j < curLocalFiles.length; ++j) {
if (!localizedFiles.contains(curLocalFiles[j])) {
// found one that wasn't there before: delete it
boolean deleted = false;
try {
if (curFC != null) {
// this is recursive, unlike File delete():
deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true);
if (curLocalFiles != null) {
for (int j = 0; j < curLocalFiles.length; ++j) {
if (!localizedFiles.contains(curLocalFiles[j])) {
// found one that wasn't there before: delete it
boolean deleted = false;
try {
if (curFC != null) {
// this is recursive, unlike File delete():
deleted =
curFC.delete(new Path(curLocalFiles[j].getName()), true);
}
} catch (IOException e) {
deleted = false;
}
if (!deleted) {
LOG.warn("Unable to delete unexpected local file/dir "
+ curLocalFiles[j].getName()
+ ": insufficient permissions?");
}
} catch (IOException e) {
deleted = false;
}
if (!deleted) {
LOG.warn("Unable to delete unexpected local file/dir "
+ curLocalFiles[j].getName() + ": insufficient permissions?");
}
}
}
}
} // end EventHandler
/**

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
View File

@ -572,13 +572,15 @@ public class MRAppMaster extends CompositeService {
private boolean isJobNamePatternMatch(JobConf conf, String jobTempDir) {
// Matched staging files should be preserved after job is finished.
if (conf.getKeepTaskFilesPattern() != null && jobTempDir != null) {
String jobFileName = Paths.get(jobTempDir).getFileName().toString();
Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
Matcher matcher = pattern.matcher(jobFileName);
return matcher.find();
} else {
return false;
java.nio.file.Path pathName = Paths.get(jobTempDir).getFileName();
if (pathName != null) {
String jobFileName = pathName.toString();
Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
Matcher matcher = pattern.matcher(jobFileName);
return matcher.find();
}
}
return false;
}
private boolean isKeepFailedTaskFiles(JobConf conf) {

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
View File

@ -98,7 +98,7 @@ class JVMId {
int jobComp = this.jobId.compareTo(that.jobId);
if(jobComp == 0) {
if(this.isMap == that.isMap) {
return Long.valueOf(this.jvmId).compareTo(that.jvmId);
return Long.compare(this.jvmId, that.jvmId);
} else {
return this.isMap ? -1 : 1;
}

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
View File

@ -34,12 +34,20 @@ public enum Operation {
KILL_TASK(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB),
SET_JOB_PRIORITY(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB),
SUBMIT_JOB(QueueACL.SUBMIT_JOB, null);
public QueueACL qACLNeeded;
public JobACL jobACLNeeded;
private final QueueACL qACLNeeded;
private final JobACL jobACLNeeded;
Operation(QueueACL qACL, JobACL jobACL) {
this.qACLNeeded = qACL;
this.jobACLNeeded = jobACL;
}
public QueueACL getqACLNeeded() {
return qACLNeeded;
}
public JobACL getJobACLNeeded() {
return jobACLNeeded;
}
}

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
View File

@ -407,17 +407,12 @@ public class HistoryFileManager extends AbstractService {
}
JobId jobId = jobIndexInfo.getJobId();
List<Path> paths = new ArrayList<Path>(2);
if (historyFile == null) {
LOG.info("No file for job-history with " + jobId + " found in cache!");
} else {
paths.add(historyFile);
}
if (confFile == null) {
LOG.info("No file for jobConf with " + jobId + " found in cache!");
} else {
paths.add(confFile);
}
if (summaryFile == null || !intermediateDoneDirFc.util().exists(

8
hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
View File

@ -67,8 +67,12 @@ public final class Parser {
private void parse(File f, Map<Parameter, List<TaskResult>> sums) throws IOException {
if (f.isDirectory()) {
println("Process directory " + f);
for(File child : f.listFiles())
parse(child, sums);
File[] files = f.listFiles();
if (files != null) {
for(File child : files) {
parse(child, sums);
}
}
} else if (f.getName().endsWith(".txt")) {
println("Parse file " + f);
final Map<Parameter, List<TaskResult>> m = new TreeMap<Parameter, List<TaskResult>>();