Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge -r 1390556:1390557 from trunk. FIXES: MAPREDUCE-4647 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1390563 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas Graves 2012-09-26 15:25:05 +00:00
parent ff5f50532a
commit e5e1887092
20 changed files with 203 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -422,6 +422,9 @@ Release 0.23.4 - UNRELEASED
BUG FIXES
MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory
in it. (Robert Evans via tgraves)
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES

11
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
View File

@ -45,6 +45,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.MapReduceChildJVM;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapred.Task;
@ -610,10 +611,12 @@ private static ContainerLaunchContext createCommonContainerLaunchContext(
if (jobJar != null) {
Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
.getUri(), remoteFS.getWorkingDirectory());
localResources.put(
MRJobConfig.JOB_JAR,
createLocalResource(remoteFS, remoteJobJar,
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION));
LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
rc.setPattern(pattern);
localResources.put(MRJobConfig.JOB_JAR, rc);
LOG.info("The job-jar file on the remote FS is "
+ remoteJobJar.toUri().toASCIIString());
} else {

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
View File

@ -152,6 +152,10 @@ public void setup(JobConf conf) throws IOException {
localArchives.add(pathString);
} else if (resource.getType() == LocalResourceType.FILE) {
localFiles.add(pathString);
} else if (resource.getType() == LocalResourceType.PATTERN) {
//PATTERN is not currently used in local mode
throw new IllegalArgumentException("Resource type PATTERN is not " +
"implemented yet. " + resource.getResource());
}
Path resourcePath;
try {

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
View File

@ -210,7 +210,7 @@ public static void setClasspath(Map<String, String> environment,
Apps.addToEnvironment(
environment,
Environment.CLASSPATH.name(),
MRJobConfig.JOB_JAR + Path.SEPARATOR);
MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR);
Apps.addToEnvironment(
environment,
Environment.CLASSPATH.name(),
@ -281,7 +281,7 @@ public static void setupDistributedCache(
}
private static String getResourceDescription(LocalResourceType type) {
if(type == LocalResourceType.ARCHIVE) {
if(type == LocalResourceType.ARCHIVE || type == LocalResourceType.PATTERN) {
return "cache archive (" + MRJobConfig.CACHE_ARCHIVES + ") ";
}
return "cache file (" + MRJobConfig.CACHE_FILES + ") ";

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
View File

@ -166,7 +166,7 @@ public class TestMRApps {
}
String env_str = env.get("CLASSPATH");
assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
env_str.indexOf("$PWD:job.jar/:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0);
env_str.indexOf("$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0);
}
@Test public void testSetClasspathWithNoUserPrecendence() {
@ -180,7 +180,7 @@ public class TestMRApps {
}
String env_str = env.get("CLASSPATH");
int index =
env_str.indexOf("job.jar/:job.jar/classes/:job.jar/lib/*:$PWD/*");
env_str.indexOf("job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*");
assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not"
+ " in the classpath!", index, -1);
assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
View File

@ -346,9 +346,13 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
jobConfPath, LocalResourceType.FILE));
if (jobConf.get(MRJobConfig.JAR) != null) {
Path jobJarPath = new Path(jobConf.get(MRJobConfig.JAR));
localResources.put(MRJobConfig.JOB_JAR,
createApplicationResource(defaultFileContext,
jobJarPath, LocalResourceType.ARCHIVE));
LocalResource rc = createApplicationResource(defaultFileContext,
jobJarPath,
LocalResourceType.PATTERN);
String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
rc.setPattern(pattern);
localResources.put(MRJobConfig.JOB_JAR, rc);
} else {
// Job jar may be null. For e.g, for pipes, the job jar is the hadoop
// mapreduce jar itself which is already on the classpath.

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
View File

@ -106,4 +106,20 @@ public interface LocalResource {
* localized
*/
public void setVisibility(LocalResourceVisibility visibility);
/**
* Get the <em>pattern</em> that should be used to extract entries from the
* archive (only used when type is <code>PATTERN</code>).
* @return <em>pattern</em> that should be used to extract entries from the
* archive.
*/
public String getPattern();
/**
* Set the <em>pattern</em> that should be used to extract entries from the
* archive (only used when type is <code>PATTERN</code>).
* @param pattern <em>pattern</em> that should be used to extract entries
* from the archive.
*/
public void setPattern(String pattern);
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
View File

@ -55,5 +55,14 @@ public enum LocalResourceType {
/**
* Regular file i.e. uninterpreted bytes.
*/
FILE
FILE,
/**
* A hybrid between archive and file. Only part of the file is unarchived,
* and the original file is left in place, but in the same directory as the
* unarchived part. The part that is unarchived is determined by pattern
* in #{@link LocalResource}. Currently only jars support pattern, all
* others will be treated like a #{@link LocalResourceType#ARCHIVE}.
*/
PATTERN
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
View File

@ -151,6 +151,25 @@ public synchronized void setVisibility(LocalResourceVisibility visibility) {
}
builder.setVisibility(convertToProtoFormat(visibility));
}
@Override
public synchronized String getPattern() {
LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasPattern()) {
return null;
}
return p.getPattern();
}
@Override
public synchronized void setPattern(String pattern) {
maybeInitBuilder();
if (pattern == null) {
builder.clearPattern();
return;
}
builder.setPattern(pattern);
}
private LocalResourceTypeProto convertToProtoFormat(LocalResourceType e) {
return ProtoUtils.convertToProtoFormat(e);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -135,6 +135,7 @@ enum LocalResourceVisibilityProto {
enum LocalResourceTypeProto {
ARCHIVE = 1;
FILE = 2;
PATTERN = 3;
}
message LocalResourceProto {
@ -142,7 +143,8 @@ message LocalResourceProto {
optional int64 size = 2;
optional int64 timestamp = 3;
optional LocalResourceTypeProto type = 4;
optional LocalResourceVisibilityProto visibility= 5;
optional LocalResourceVisibilityProto visibility = 5;
optional string pattern = 6;
}
message ApplicationResourceUsageReportProto {

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
View File

@ -25,6 +25,7 @@
import java.security.PrivilegedExceptionAction;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -103,9 +104,9 @@ private Path copy(Path sCopy, Path dstdir) throws IOException {
return dCopy;
}
private long unpack(File localrsrc, File dst) throws IOException {
private long unpack(File localrsrc, File dst, Pattern pattern) throws IOException {
switch (resource.getType()) {
case ARCHIVE:
case ARCHIVE: {
String lowerDst = dst.getName().toLowerCase();
if (lowerDst.endsWith(".jar")) {
RunJar.unJar(localrsrc, dst);
@ -122,7 +123,39 @@ private long unpack(File localrsrc, File dst) throws IOException {
+ "] to [" + dst + "]");
}
}
break;
}
break;
case PATTERN: {
String lowerDst = dst.getName().toLowerCase();
if (lowerDst.endsWith(".jar")) {
RunJar.unJar(localrsrc, dst, pattern);
File newDst = new File(dst, dst.getName());
if (!dst.exists() && !dst.mkdir()) {
throw new IOException("Unable to create directory: [" + dst + "]");
}
if (!localrsrc.renameTo(newDst)) {
throw new IOException("Unable to rename file: [" + localrsrc
+ "] to [" + newDst + "]");
}
} else if (lowerDst.endsWith(".zip")) {
LOG.warn("Treating [" + localrsrc + "] as an archive even though it " +
"was specified as PATTERN");
FileUtil.unZip(localrsrc, dst);
} else if (lowerDst.endsWith(".tar.gz") ||
lowerDst.endsWith(".tgz") ||
lowerDst.endsWith(".tar")) {
LOG.warn("Treating [" + localrsrc + "] as an archive even though it " +
"was specified as PATTERN");
FileUtil.unTar(localrsrc, dst);
} else {
LOG.warn("Cannot unpack " + localrsrc);
if (!localrsrc.renameTo(dst)) {
throw new IOException("Unable to rename file: [" + localrsrc
+ "] to [" + dst + "]");
}
}
}
break;
case FILE:
default:
if (!localrsrc.renameTo(dst)) {
@ -163,8 +196,13 @@ public Path call() throws Exception {
public Path run() throws Exception {
return files.makeQualified(copy(sCopy, dst_work));
};
});
unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
});
Pattern pattern = null;
String p = resource.getPattern();
if(p != null) {
pattern = Pattern.compile(p);
}
unpack(new File(dTmp.toUri()), new File(dFinal.toUri()), pattern);
changePermissions(dFinal.getFileSystem(conf), dFinal);
files.rename(dst_work, destDirPath, Rename.OVERWRITE);
} catch (Exception e) {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
View File

@ -107,8 +107,9 @@ static LocalResource createJar(FileContext files, Path p,
FileStatus status = files.getFileStatus(p);
ret.setSize(status.getLen());
ret.setTimestamp(status.getModificationTime());
ret.setType(LocalResourceType.ARCHIVE);
ret.setType(LocalResourceType.PATTERN);
ret.setVisibility(vis);
ret.setPattern("classes/.*");
return ret;
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -209,6 +209,7 @@ static long getEstimatedSize(LocalResource rsrc) {
}
switch (rsrc.getType()) {
case ARCHIVE:
case PATTERN:
return 5 * rsrc.getSize();
case FILE:
default:

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
View File

@ -34,6 +34,7 @@ public class LocalResourceRequest
private final long timestamp;
private final LocalResourceType type;
private final LocalResourceVisibility visibility;
private final String pattern;
/**
* Wrap API resource to match against cache of localized resources.
@ -45,22 +46,28 @@ public LocalResourceRequest(LocalResource resource)
this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
resource.getTimestamp(),
resource.getType(),
resource.getVisibility());
resource.getVisibility(),
resource.getPattern());
}
LocalResourceRequest(Path loc, long timestamp, LocalResourceType type,
LocalResourceVisibility visibility) {
LocalResourceVisibility visibility, String pattern) {
this.loc = loc;
this.timestamp = timestamp;
this.type = type;
this.visibility = visibility;
this.pattern = pattern;
}
@Override
public int hashCode() {
return loc.hashCode() ^
int hash = loc.hashCode() ^
(int)((timestamp >>> 32) ^ timestamp) *
type.hashCode();
if(pattern != null) {
hash = hash ^ pattern.hashCode();
}
return hash;
}
@Override
@ -72,9 +79,14 @@ public boolean equals(Object o) {
return false;
}
final LocalResourceRequest other = (LocalResourceRequest) o;
String pattern = getPattern();
String otherPattern = other.getPattern();
boolean patternEquals = (pattern == null && otherPattern == null) ||
(pattern != null && otherPattern != null && pattern.equals(otherPattern));
return getPath().equals(other.getPath()) &&
getTimestamp() == other.getTimestamp() &&
getType() == other.getType();
getType() == other.getType() &&
patternEquals;
}
@Override
@ -87,6 +99,19 @@ public int compareTo(LocalResourceRequest other) {
ret = (int)(getTimestamp() - other.getTimestamp());
if (0 == ret) {
ret = getType().ordinal() - other.getType().ordinal();
if (0 == ret) {
String pattern = getPattern();
String otherPattern = other.getPattern();
if (pattern == null && otherPattern == null) {
ret = 0;
} else if (pattern == null) {
ret = -1;
} else if (otherPattern == null) {
ret = 1;
} else {
ret = pattern.compareTo(otherPattern);
}
}
}
}
return ret;
@ -121,6 +146,11 @@ public LocalResourceVisibility getVisibility() {
return visibility;
}
@Override
public String getPattern() {
return pattern;
}
@Override
public void setResource(URL resource) {
throw new UnsupportedOperationException();
@ -145,14 +175,20 @@ public void setType(LocalResourceType type) {
public void setVisibility(LocalResourceVisibility visibility) {
throw new UnsupportedOperationException();
}
@Override
public void setPattern(String pattern) {
throw new UnsupportedOperationException();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{ ");
sb.append(getPath().toString()).append(", ");
sb.append(getTimestamp()).append(", ");
sb.append(getType()).append(" }");
sb.append(getType()).append(", ");
sb.append(getPattern()).append(" }");
return sb.toString();
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
View File

@ -219,7 +219,8 @@ public void transition(LocalizedResource rsrc, ResourceEvent event) {
ContainerId container = ctxt.getContainerId();
rsrc.ref.add(container);
rsrc.dispatcher.getEventHandler().handle(
new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt));
new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt,
req.getLocalResourceRequest().getPattern()));
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
View File

@ -752,6 +752,7 @@ private LocalResource findNextResource() {
next.setTimestamp(nextRsrc.getTimestamp());
next.setType(nextRsrc.getType());
next.setVisibility(evt.getVisibility());
next.setPattern(evt.getPattern());
scheduled.put(nextRsrc, evt);
return next;
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
View File

@ -32,14 +32,16 @@ public class LocalizerResourceRequestEvent extends LocalizerEvent {
private final LocalizerContext context;
private final LocalizedResource resource;
private final LocalResourceVisibility vis;
private final String pattern;
public LocalizerResourceRequestEvent(LocalizedResource resource,
LocalResourceVisibility vis, LocalizerContext context) {
LocalResourceVisibility vis, LocalizerContext context, String pattern) {
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
ConverterUtils.toString(context.getContainerId()));
this.vis = vis;
this.context = context;
this.resource = resource;
this.pattern = pattern;
}
public LocalizedResource getResource() {
@ -54,4 +56,8 @@ public LocalResourceVisibility getVisibility() {
return vis;
}
public String getPattern() {
return pattern;
}
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
View File

@ -37,7 +37,7 @@
public class TestLocalResource {
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
long timestamp, LocalResourceType type, LocalResourceVisibility state)
long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
throws URISyntaxException {
org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
@ -45,6 +45,7 @@ static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p,
ret.setTimestamp(timestamp);
ret.setType(type);
ret.setVisibility(state);
ret.setPattern(pattern);
return ret;
}
@ -72,9 +73,9 @@ public void testResourceEquality() throws URISyntaxException {
long basetime = r.nextLong() >>> 2;
org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null);
org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null);
final LocalResourceRequest a = new LocalResourceRequest(yA);
LocalResourceRequest b = new LocalResourceRequest(yA);
checkEqual(a, b);
@ -83,31 +84,37 @@ public void testResourceEquality() throws URISyntaxException {
// ignore visibility
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE);
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE, null);
b = new LocalResourceRequest(yB);
checkEqual(a, b);
// ignore size
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE);
new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE, null);
b = new LocalResourceRequest(yB);
checkEqual(a, b);
// note path
yB = getYarnResource(
new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC);
new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null);
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
// note type
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC);
new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null);
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
// note timestamp
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC);
new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, null);
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
// note pattern
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, "^/foo/.*");
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
}
@ -120,24 +127,35 @@ public void testResourceOrder() throws URISyntaxException {
System.out.println("SEED: " + seed);
long basetime = r.nextLong() >>> 2;
org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, "^/foo/.*");
final LocalResourceRequest a = new LocalResourceRequest(yA);
// Path primary
org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC);
new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC, "^/foo/.*");
LocalResourceRequest b = new LocalResourceRequest(yB);
assertTrue(0 > a.compareTo(b));
// timestamp secondary
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC);
new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC, "^/foo/.*");
b = new LocalResourceRequest(yB);
assertTrue(0 > a.compareTo(b));
// type tertiary
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC);
new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/foo/.*");
b = new LocalResourceRequest(yB);
assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
// path 4th
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/food/.*");
b = new LocalResourceRequest(yB);
assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
yB = getYarnResource(
new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, null);
b = new LocalResourceRequest(yB);
assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
View File

@ -230,7 +230,7 @@ private LocalResourceRequest createLocalResourceRequest(String user, int i,
long ts, LocalResourceVisibility vis) {
final LocalResourceRequest req =
new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i),
ts + i * 2000, LocalResourceType.FILE, vis);
ts + i * 2000, LocalResourceType.FILE, vis, null);
return req;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
View File

@ -83,7 +83,7 @@ LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
for (int i = 0; i < nRsrcs; ++i) {
final LocalResourceRequest req = new LocalResourceRequest(
new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC);
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
final long ts = timestamp + i * tsstep;
final Path p = new Path("file:///local/" + user + "/rsrc" + i);
LocalizedResource rsrc = new LocalizedResource(req, null) {