Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8584. Several typos in Log Aggregation related classes. Contributed by Szilard Nemeth.

This commit is contained in:
bibinchundatt 2018-07-30 23:25:19 +05:30
parent e8f952ef06
commit 2b39ad2698
10 changed files with 35 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
View File

@ -258,7 +258,7 @@ private void scheduleLogDeletionTask() throws IOException {
return;
}
setLogAggCheckIntervalMsecs(retentionSecs);
task = new LogDeletionTask(conf, retentionSecs, creatRMClient());
task = new LogDeletionTask(conf, retentionSecs, createRMClient());
timer = new Timer();
timer.scheduleAtFixedRate(task, 0, checkIntervalMsecs);
}
@ -281,7 +281,7 @@ protected Configuration createConf() {
// We have already marked ApplicationClientProtocol.getApplicationReport
// as @Idempotent, it will automatically take care of RM restart/failover.
@VisibleForTesting
protected ApplicationClientProtocol creatRMClient() throws IOException {
protected ApplicationClientProtocol createRMClient() throws IOException {
return ClientRMProxy.createRMProxy(getConfig(),
ApplicationClientProtocol.class);
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
View File

@ -178,7 +178,7 @@ public static class LogValue {
* The set of log files that are older than retention policy that will
* not be uploaded but ready for deletion.
*/
private final Set<File> obseleteRetentionLogFiles = new HashSet<File>();
private final Set<File> obsoleteRetentionLogFiles = new HashSet<File>();
// TODO Maybe add a version string here. Instead of changing the version of
// the entire k-v format
@ -324,7 +324,7 @@ private Set<File> getPendingLogFilesToUpload(File containerLogDir) {
// if log files are older than retention policy, do not upload them.
// but schedule them for deletion.
if(logRetentionContext != null && !logRetentionContext.shouldRetainLog()){
obseleteRetentionLogFiles.addAll(candidates);
obsoleteRetentionLogFiles.addAll(candidates);
candidates.clear();
return candidates;
}
@ -396,9 +396,9 @@ public Set<String> getCurrentUpLoadedFileMeta() {
return info;
}
public Set<Path> getObseleteRetentionLogFiles() {
public Set<Path> getObsoleteRetentionLogFiles() {
Set<Path> path = new HashSet<Path>();
for(File file: this.obseleteRetentionLogFiles) {
for(File file: this.obsoleteRetentionLogFiles) {
path.add(new Path(file.getAbsolutePath()));
}
return path;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
View File

@ -115,16 +115,16 @@ public LogAggregationFileController() {}
*/
public void initialize(Configuration conf, String controllerName) {
this.conf = conf;
int configuredRentionSize = conf.getInt(
int configuredRetentionSize = conf.getInt(
YarnConfiguration.NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP,
YarnConfiguration
.DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP);
if (configuredRentionSize <= 0) {
if (configuredRetentionSize <= 0) {
this.retentionSize =
YarnConfiguration
.DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP;
} else {
this.retentionSize = configuredRentionSize;
this.retentionSize = configuredRetentionSize;
}
this.fileControllerName = controllerName;
initInternal(conf);

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
View File

@ -187,8 +187,8 @@ protected void render(Block html) {
FSDataInputStream fsin = fileContext.open(thisNodeFile.getPath());
int bufferSize = 65536;
for (IndexedFileLogMeta candidate : candidates) {
if (candidate.getLastModificatedTime() < startTime
|| candidate.getLastModificatedTime() > endTime) {
if (candidate.getLastModifiedTime() < startTime
|| candidate.getLastModifiedTime() > endTime) {
continue;
}
byte[] cbuf = new byte[bufferSize];
@ -205,7 +205,7 @@ protected void render(Block html) {
html.pre().__("\n\n").__();
html.p().__("Log Type: " + candidate.getFileName()).__();
html.p().__("Log Upload Time: " + Times.format(
candidate.getLastModificatedTime())).__();
candidate.getLastModifiedTime())).__();
html.p().__("Log Length: " + Long.toString(
logLength)).__();
long startIndex = start < 0

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
View File

@ -404,7 +404,7 @@ public void write(LogKey logKey, LogValue logValue) throws IOException {
meta.setStartIndex(outputStreamState.getStartPos());
meta.setFileSize(fileLength);
}
meta.setLastModificatedTime(logFile.lastModified());
meta.setLastModifiedTime(logFile.lastModified());
metas.add(meta);
}
logsMetaInThisCycle.addContainerLogMeta(containerId, metas);
@ -499,12 +499,12 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest,
.getRemoteNodeFileDir(conf, appId, logRequest.getAppOwner(),
this.remoteRootLogDir, this.remoteRootLogDirSuffix);
if (!nodeFiles.hasNext()) {
throw new IOException("There is no available log fils for "
throw new IOException("There is no available log file for "
+ "application:" + appId);
}
List<FileStatus> allFiles = getAllNodeFiles(nodeFiles, appId);
if (allFiles.isEmpty()) {
throw new IOException("There is no available log fils for "
throw new IOException("There is no available log file for "
+ "application:" + appId);
}
Map<String, Long> checkSumFiles = parseCheckSumFiles(allFiles);
@ -581,7 +581,7 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest,
decompressor, getFSInputBufferSize(conf));
LogToolUtils.outputContainerLog(candidate.getContainerId(),
nodeName, candidate.getFileName(), candidate.getFileSize(), size,
Times.format(candidate.getLastModificatedTime()),
Times.format(candidate.getLastModifiedTime()),
in, os, buf, ContainerLogAggregationType.AGGREGATED);
byte[] b = aggregatedLogSuffix(candidate.getFileName())
.getBytes(Charset.forName("UTF-8"));
@ -618,12 +618,12 @@ public List<ContainerLogMeta> readAggregatedLogsMeta(
.getRemoteNodeFileDir(conf, appId, appOwner, this.remoteRootLogDir,
this.remoteRootLogDirSuffix);
if (!nodeFiles.hasNext()) {
throw new IOException("There is no available log fils for "
throw new IOException("There is no available log file for "
+ "application:" + appId);
}
List<FileStatus> allFiles = getAllNodeFiles(nodeFiles, appId);
if (allFiles.isEmpty()) {
throw new IOException("There is no available log fils for "
throw new IOException("There is no available log file for "
+ "application:" + appId);
}
Map<String, Long> checkSumFiles = parseCheckSumFiles(allFiles);
@ -660,7 +660,7 @@ public List<ContainerLogMeta> readAggregatedLogsMeta(
for (IndexedFileLogMeta aMeta : log.getValue()) {
meta.addLogMeta(aMeta.getFileName(), Long.toString(
aMeta.getFileSize()),
Times.format(aMeta.getLastModificatedTime()));
Times.format(aMeta.getLastModifiedTime()));
}
containersLogMeta.add(meta);
}
@ -671,7 +671,7 @@ public List<ContainerLogMeta> readAggregatedLogsMeta(
logMeta.getContainerLogMeta(containerIdStr)) {
meta.addLogMeta(log.getFileName(), Long.toString(
log.getFileSize()),
Times.format(log.getLastModificatedTime()));
Times.format(log.getLastModifiedTime()));
}
containersLogMeta.add(meta);
}
@ -1002,7 +1002,7 @@ public static class IndexedFileLogMeta implements Serializable {
private String fileName;
private long fileSize;
private long fileCompressedSize;
private long lastModificatedTime;
private long lastModifiedTime;
private long startIndex;
public String getFileName() {
@ -1026,11 +1026,11 @@ public void setFileCompressedSize(long fileCompressedSize) {
this.fileCompressedSize = fileCompressedSize;
}
public long getLastModificatedTime() {
return lastModificatedTime;
public long getLastModifiedTime() {
return lastModifiedTime;
}
public void setLastModificatedTime(long lastModificatedTime) {
this.lastModificatedTime = lastModificatedTime;
public void setLastModifiedTime(long lastModifiedTime) {
this.lastModifiedTime = lastModifiedTime;
}
public long getStartIndex() {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
View File

@ -275,7 +275,7 @@ public List<ContainerLogMeta> readAggregatedLogsMeta(
RemoteIterator<FileStatus> nodeFiles = LogAggregationUtils
.getRemoteNodeFileDir(conf, appId, appOwner);
if (nodeFiles == null) {
throw new IOException("There is no available log fils for "
throw new IOException("There is no available log file for "
+ "application:" + appId);
}
while (nodeFiles.hasNext()) {

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java
View File

@ -160,7 +160,7 @@ public void testDeletion() throws Exception {
AggregatedLogDeletionService deletionService =
new AggregatedLogDeletionService() {
@Override
protected ApplicationClientProtocol creatRMClient()
protected ApplicationClientProtocol createRMClient()
throws IOException {
try {
return createMockRMClient(finishedApplications,
@ -262,7 +262,7 @@ protected Configuration createConf() {
return conf;
}
@Override
protected ApplicationClientProtocol creatRMClient()
protected ApplicationClientProtocol createRMClient()
throws IOException {
try {
return createMockRMClient(finishedApplications, null);
@ -353,7 +353,7 @@ public void testCheckInterval() throws Exception {
AggregatedLogDeletionService deletionSvc =
new AggregatedLogDeletionService() {
@Override
protected ApplicationClientProtocol creatRMClient()
protected ApplicationClientProtocol createRMClient()
throws IOException {
try {
return createMockRMClient(finishedApplications, null);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
View File

@ -632,7 +632,7 @@ public boolean apply(String next) {
// need to return files uploaded or older-than-retention clean up.
return Sets.union(logValue.getCurrentUpLoadedFilesPath(),
logValue.getObseleteRetentionLogFiles());
logValue.getObsoleteRetentionLogFiles());
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
View File

@ -117,9 +117,9 @@ protected void serviceInit(Configuration conf) throws Exception {
LOG.info("Log aggregation debug mode enabled. rollingMonitorInterval = "
+ rollingMonitorInterval);
} else {
LOG.warn("rollingMonitorIntervall should be more than or equal to "
+ MIN_LOG_ROLLING_INTERVAL + " seconds. Using "
+ MIN_LOG_ROLLING_INTERVAL + " seconds instead.");
LOG.warn("rollingMonitorInterval should be more than or equal to {} " +
"seconds. Using {} seconds instead.",
MIN_LOG_ROLLING_INTERVAL, MIN_LOG_ROLLING_INTERVAL);
this.rollingMonitorInterval = MIN_LOG_ROLLING_INTERVAL;
}
} else if (rollingMonitorInterval <= 0) {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java
View File

@ -110,7 +110,7 @@ public void updateLogAggregationStatus(ApplicationId appId,
LogAggregationStatus logAggregationStatus, long updateTime,
String diagnosis, boolean finalized) {
if (disabled) {
LOG.warn("The log aggregation is diabled. No need to update "
LOG.warn("The log aggregation is disabled. No need to update "
+ "the log aggregation status");
}
// In NM, each application has exactly one appLogAggregator thread
@ -164,7 +164,7 @@ public void updateLogAggregationStatus(ApplicationId appId,
public List<LogAggregationReport> pullCachedLogAggregationReports() {
List<LogAggregationReport> reports = new ArrayList<>();
if (disabled) {
LOG.warn("The log aggregation is diabled."
LOG.warn("The log aggregation is disabled."
+ "There is no cached log aggregation status.");
return reports;
}