merge -r 1301730:1301731 from trunk to branch-0.23. FIXES: MAPREDUCE-4006
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1301734 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
1514466b7a
commit
eb0311aedc
|
@ -240,6 +240,9 @@ Release 0.23.2 - UNRELEASED
|
|||
MAPREDUCE-4005. Fixes broken AM container logs URL on ResourceManager
|
||||
Application Info page. (Jason Lowe via sseth)
|
||||
|
||||
MAPREDUCE-4006. History server container log web UI sometimes combines
|
||||
stderr/stdout/syslog contents together (Siddharth Seth via tgraves)
|
||||
|
||||
Release 0.23.1 - 2012-02-17
|
||||
|
||||
NEW FEATURES
|
||||
|
|
|
@ -430,7 +430,7 @@ public class AggregatedLogFormat {
|
|||
int currentRead = 0;
|
||||
int totalRead = 0;
|
||||
while ((currentRead = reader.read(cbuf, 0, bufferSize)) != -1) {
|
||||
writer.write(cbuf);
|
||||
writer.write(cbuf, 0, currentRead);
|
||||
totalRead += currentRead;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,144 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.logaggregation;
|
||||
|
||||
import java.io.DataInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.Collections;
|
||||
|
||||
import junit.framework.Assert;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter;
|
||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestAggregatedLogFormat {
|
||||
|
||||
private static final File testWorkDir = new File("target",
|
||||
"TestAggregatedLogFormat");
|
||||
private static final Configuration conf = new Configuration();
|
||||
private static final FileSystem fs;
|
||||
private static final char filler = 'x';
|
||||
private static final Log LOG = LogFactory
|
||||
.getLog(TestAggregatedLogFormat.class);
|
||||
|
||||
static {
|
||||
try {
|
||||
fs = FileSystem.get(conf);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Before
|
||||
@After
|
||||
public void cleanupTestDir() throws Exception {
|
||||
Path workDirPath = new Path(testWorkDir.getAbsolutePath());
|
||||
LOG.info("Cleaning test directory [" + workDirPath + "]");
|
||||
fs.delete(workDirPath, true);
|
||||
}
|
||||
|
||||
//Verify the output generated by readAContainerLogs(DataInputStream, Writer)
|
||||
@Test
|
||||
public void testReadAcontainerLogs1() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
|
||||
Path remoteAppLogFile =
|
||||
new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
|
||||
Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
|
||||
ContainerId testContainerId = BuilderUtils.newContainerId(1, 1, 1, 1);
|
||||
Path t =
|
||||
new Path(srcFileRoot, testContainerId.getApplicationAttemptId()
|
||||
.getApplicationId().toString());
|
||||
Path srcFilePath = new Path(t, testContainerId.toString());
|
||||
|
||||
int numChars = 80000;
|
||||
|
||||
writeSrcFile(srcFilePath, "stdout", numChars);
|
||||
|
||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||
LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);
|
||||
|
||||
LogKey logKey = new LogKey(testContainerId);
|
||||
LogValue logValue =
|
||||
new LogValue(Collections.singletonList(srcFileRoot.toString()),
|
||||
testContainerId);
|
||||
|
||||
logWriter.append(logKey, logValue);
|
||||
logWriter.closeWriter();
|
||||
|
||||
LogReader logReader = new LogReader(conf, remoteAppLogFile);
|
||||
LogKey rLogKey = new LogKey();
|
||||
DataInputStream dis = logReader.next(rLogKey);
|
||||
Writer writer = new StringWriter();
|
||||
LogReader.readAcontainerLogs(dis, writer);
|
||||
|
||||
String s = writer.toString();
|
||||
int expectedLength =
|
||||
"\n\nLogType:stdout".length() + ("\nLogLength:" + numChars).length()
|
||||
+ "\nLog Contents:\n".length() + numChars;
|
||||
Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
|
||||
Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
|
||||
Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0 ; i < numChars ; i++) {
|
||||
sb.append(filler);
|
||||
}
|
||||
String expectedContent = sb.toString();
|
||||
Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
|
||||
|
||||
Assert.assertEquals(expectedLength, s.length());
|
||||
}
|
||||
|
||||
private void writeSrcFile(Path srcFilePath, String fileName, long length)
|
||||
throws IOException {
|
||||
File dir = new File(srcFilePath.toString());
|
||||
if (!dir.exists()) {
|
||||
if (!dir.mkdirs()) {
|
||||
throw new IOException("Unable to create directory : " + dir);
|
||||
}
|
||||
}
|
||||
File outputFile = new File(new File(srcFilePath.toString()), fileName);
|
||||
FileOutputStream os = new FileOutputStream(outputFile);
|
||||
OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
|
||||
int ch = filler;
|
||||
for (int i = 0; i < length; i++) {
|
||||
osw.write(ch);
|
||||
}
|
||||
osw.close();
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue