YARN-7115. Move BoundedAppender to org.hadoop.yarn.util pacakge
(Contributed by Jian He via Daniel Templeton)
This commit is contained in:
Daniel Templeton
parent
3e0e2033cc
commit
cc23514aba
|
|
@ -108,6 +108,15 @@
|
|||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
</dependency>
|
||||
<!--
|
||||
junit must be before mockito-all on the classpath. mockito-all bundles its
|
||||
own copy of the hamcrest classes, but they don't match our junit version.
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-all</artifactId>
|
||||
|
|
@ -128,11 +137,6 @@
|
|||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcprov-jdk16</artifactId>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,142 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.util;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Preconditions;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
||||
/**
|
||||
* A {@link CharSequence} appender that considers its {@link #limit} as upper
|
||||
* bound.
|
||||
* <p>
|
||||
* When {@link #limit} would be reached on append, past messages will be
|
||||
* truncated from head, and a header telling the user about truncation will be
|
||||
* prepended, with ellipses in between header and messages.
|
||||
* <p>
|
||||
* Note that header and ellipses are not counted against {@link #limit}.
|
||||
* <p>
|
||||
* An example:
|
||||
*
|
||||
* <pre>
|
||||
* {@code
|
||||
* // At the beginning it's an empty string
|
||||
* final Appendable shortAppender = new BoundedAppender(80);
|
||||
* // The whole message fits into limit
|
||||
* shortAppender.append(
|
||||
* "message1 this is a very long message but fitting into limit\n");
|
||||
* // The first message is truncated, the second not
|
||||
* shortAppender.append("message2 this is shorter than the previous one\n");
|
||||
* // The first message is deleted, the second truncated, the third
|
||||
* // preserved
|
||||
* shortAppender.append("message3 this is even shorter message, maybe.\n");
|
||||
* // The first two are deleted, the third one truncated, the last preserved
|
||||
* shortAppender.append("message4 the shortest one, yet the greatest :)");
|
||||
* // Current contents are like this:
|
||||
* // Diagnostic messages truncated, showing last 80 chars out of 199:
|
||||
* // ...s is even shorter message, maybe.
|
||||
* // message4 the shortest one, yet the greatest :)
|
||||
* }
|
||||
* </pre>
|
||||
* <p>
|
||||
* Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed
|
||||
* just like in {@link StringBuilder#append(CharSequence) original
|
||||
* implementation}.
|
||||
* <p>
|
||||
* Note that this class is not thread safe.
|
||||
*/
|
||||
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Unstable
|
||||
@VisibleForTesting
|
||||
public class BoundedAppender {
|
||||
@VisibleForTesting
|
||||
public static final String TRUNCATED_MESSAGES_TEMPLATE =
|
||||
"Diagnostic messages truncated, showing last "
|
||||
+ "%d chars out of %d:%n...%s";
|
||||
|
||||
private final int limit;
|
||||
private final StringBuilder messages = new StringBuilder();
|
||||
private int totalCharacterCount = 0;
|
||||
|
||||
public BoundedAppender(final int limit) {
|
||||
Preconditions.checkArgument(limit > 0, "limit should be positive");
|
||||
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a {@link CharSequence} considering {@link #limit}, truncating
|
||||
* from the head of {@code csq} or {@link #messages} when necessary.
|
||||
*
|
||||
* @param csq the {@link CharSequence} to append
|
||||
* @return this
|
||||
*/
|
||||
public BoundedAppender append(final CharSequence csq) {
|
||||
appendAndCount(csq);
|
||||
checkAndCut();
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
private void appendAndCount(final CharSequence csq) {
|
||||
final int before = messages.length();
|
||||
messages.append(csq);
|
||||
final int after = messages.length();
|
||||
totalCharacterCount += after - before;
|
||||
}
|
||||
|
||||
private void checkAndCut() {
|
||||
if (messages.length() > limit) {
|
||||
final int newStart = messages.length() - limit;
|
||||
messages.delete(0, newStart);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current length of messages considering truncates
|
||||
* without header and ellipses.
|
||||
*
|
||||
* @return current length
|
||||
*/
|
||||
public int length() {
|
||||
return messages.length();
|
||||
}
|
||||
|
||||
public int getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a string representation of the actual contents, displaying also a
|
||||
* header and ellipses when there was a truncate.
|
||||
*
|
||||
* @return String representation of the {@link #messages}
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
if (messages.length() < totalCharacterCount) {
|
||||
return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(),
|
||||
totalCharacterCount, messages.toString());
|
||||
}
|
||||
|
||||
return messages.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -16,14 +16,13 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
|
||||
package org.apache.hadoop.yarn.util;
|
||||
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl.BoundedAppender;
|
||||
|
||||
/**
|
||||
* Test class for {@link BoundedAppender}.
|
||||
|
|
@ -38,7 +38,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
|
|||
|
||||
import javax.crypto.SecretKey;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
|
|
@ -110,6 +109,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
|
|||
import org.apache.hadoop.yarn.state.SingleArcTransition;
|
||||
import org.apache.hadoop.yarn.state.StateMachine;
|
||||
import org.apache.hadoop.yarn.state.StateMachineFactory;
|
||||
import org.apache.hadoop.yarn.util.BoundedAppender;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
|
@ -1326,7 +1326,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
// AFTER the initial saving on app-attempt-start
|
||||
// These fields can be visible from outside only after they are saved in
|
||||
// StateStore
|
||||
BoundedAppender diags = new BoundedAppender(diagnostics.limit);
|
||||
BoundedAppender diags = new BoundedAppender(diagnostics.getLimit());
|
||||
|
||||
// don't leave the tracking URL pointing to a non-existent AM
|
||||
if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
|
||||
|
|
@ -2295,114 +2295,4 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
return Collections.EMPTY_SET;
|
||||
}
|
||||
|
||||
/**
|
||||
* A {@link CharSequence} appender that considers its {@link #limit} as upper
|
||||
* bound.
|
||||
* <p>
|
||||
* When {@link #limit} would be reached on append, past messages will be
|
||||
* truncated from head, and a header telling the user about truncation will be
|
||||
* prepended, with ellipses in between header and messages.
|
||||
* <p>
|
||||
* Note that header and ellipses are not counted against {@link #limit}.
|
||||
* <p>
|
||||
* An example:
|
||||
*
|
||||
* <pre>
|
||||
* {@code
|
||||
* // At the beginning it's an empty string
|
||||
* final Appendable shortAppender = new BoundedAppender(80);
|
||||
* // The whole message fits into limit
|
||||
* shortAppender.append(
|
||||
* "message1 this is a very long message but fitting into limit\n");
|
||||
* // The first message is truncated, the second not
|
||||
* shortAppender.append("message2 this is shorter than the previous one\n");
|
||||
* // The first message is deleted, the second truncated, the third
|
||||
* // preserved
|
||||
* shortAppender.append("message3 this is even shorter message, maybe.\n");
|
||||
* // The first two are deleted, the third one truncated, the last preserved
|
||||
* shortAppender.append("message4 the shortest one, yet the greatest :)");
|
||||
* // Current contents are like this:
|
||||
* // Diagnostic messages truncated, showing last 80 chars out of 199:
|
||||
* // ...s is even shorter message, maybe.
|
||||
* // message4 the shortest one, yet the greatest :)
|
||||
* }
|
||||
* </pre>
|
||||
* <p>
|
||||
* Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed
|
||||
* just like in {@link StringBuilder#append(CharSequence) original
|
||||
* implementation}.
|
||||
* <p>
|
||||
* Note that this class is not thread safe.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static class BoundedAppender {
|
||||
@VisibleForTesting
|
||||
static final String TRUNCATED_MESSAGES_TEMPLATE =
|
||||
"Diagnostic messages truncated, showing last "
|
||||
+ "%d chars out of %d:%n...%s";
|
||||
|
||||
private final int limit;
|
||||
private final StringBuilder messages = new StringBuilder();
|
||||
private int totalCharacterCount = 0;
|
||||
|
||||
BoundedAppender(final int limit) {
|
||||
Preconditions.checkArgument(limit > 0, "limit should be positive");
|
||||
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a {@link CharSequence} considering {@link #limit}, truncating
|
||||
* from the head of {@code csq} or {@link #messages} when necessary.
|
||||
*
|
||||
* @param csq the {@link CharSequence} to append
|
||||
* @return this
|
||||
*/
|
||||
BoundedAppender append(final CharSequence csq) {
|
||||
appendAndCount(csq);
|
||||
checkAndCut();
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
private void appendAndCount(final CharSequence csq) {
|
||||
final int before = messages.length();
|
||||
messages.append(csq);
|
||||
final int after = messages.length();
|
||||
totalCharacterCount += after - before;
|
||||
}
|
||||
|
||||
private void checkAndCut() {
|
||||
if (messages.length() > limit) {
|
||||
final int newStart = messages.length() - limit;
|
||||
messages.delete(0, newStart);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current length of messages considering truncates
|
||||
* without header and ellipses.
|
||||
*
|
||||
* @return current length
|
||||
*/
|
||||
int length() {
|
||||
return messages.length();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a string representation of the actual contents, displaying also a
|
||||
* header and ellipses when there was a truncate.
|
||||
*
|
||||
* @return String representation of the {@link #messages}
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
if (messages.length() < totalCharacterCount) {
|
||||
return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(),
|
||||
totalCharacterCount, messages.toString());
|
||||
}
|
||||
|
||||
return messages.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
|||
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||
import org.apache.hadoop.yarn.util.BoundedAppender;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
|
|
@ -86,7 +87,7 @@ public class TestRMAppAttemptImplDiagnostics {
|
|||
appAttempt.appendDiagnostics(beyondLimit);
|
||||
|
||||
final String truncated = String.format(
|
||||
RMAppAttemptImpl.BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024,
|
||||
BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024,
|
||||
1025, beyondLimit.substring(1));
|
||||
|
||||
assertEquals("messages beyond limit should be truncated", truncated,
|
||||
|
|
|
|||
Loading…
Reference in New Issue