YARN-2165. Added the sanity check for the numeric configuration values of the timeline service. Contributed by Vasanth kumar RJ.

This commit is contained in:
Zhijie Shen 2014-11-26 12:09:34 -08:00
parent 978736d486
commit 8f1454cc6d
8 changed files with 161 additions and 2 deletions

View File

@ -98,6 +98,9 @@ Release 2.7.0 - UNRELEASED
YARN-2404. Removed ApplicationAttemptState and ApplicationState class in YARN-2404. Removed ApplicationAttemptState and ApplicationState class in
RMStateStore. (Tsuyoshi OZAWA via jianhe) RMStateStore. (Tsuyoshi OZAWA via jianhe)
YARN-2165. Added the sanity check for the numeric configuration values of
the timeline service. (Vasanth kumar RJ via zjshen)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -67,6 +67,7 @@ import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientRequest; import com.sun.jersey.api.client.ClientRequest;
@ -141,6 +142,18 @@ public class TimelineClientImpl extends TimelineClient {
// Constructor with default retry settings // Constructor with default retry settings
public TimelineClientConnectionRetry(Configuration conf) { public TimelineClientConnectionRetry(Configuration conf) {
Preconditions.checkArgument(conf.getInt(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES) >= -1,
"%s property value should be greater than or equal to -1",
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
Preconditions
.checkArgument(
conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS) > 0,
"%s property value should be greater than zero",
YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS);
maxRetries = conf.getInt( maxRetries = conf.getInt(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES); YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES);

View File

@ -1355,7 +1355,8 @@
<property> <property>
<description> <description>
Default maximum number of retires for timeline servive client. Default maximum number of retires for timeline servive client
and value -1 means no limit.
</description> </description>
<name>yarn.timeline-service.client.max-retries</name> <name>yarn.timeline-service.client.max-retries</name>
<value>30</value> <value>30</value>

View File

@ -154,6 +154,29 @@ public class TestTimelineClient {
@Test @Test
public void testCheckRetryCount() throws Exception { public void testCheckRetryCount() throws Exception {
try {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
-2);
createTimelineClient(conf);
Assert.fail();
} catch(IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES));
}
try {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
0);
createTimelineClient(conf);
Assert.fail();
} catch(IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS));
}
int newMaxRetries = 5; int newMaxRetries = 5;
long newIntervalMs = 500; long newIntervalMs = 500;
YarnConfiguration conf = new YarnConfiguration(); YarnConfiguration conf = new YarnConfiguration();

View File

@ -61,6 +61,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider; import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider;
import com.google.common.base.Preconditions;
public class ApplicationHistoryClientService extends AbstractService { public class ApplicationHistoryClientService extends AbstractService {
private static final Log LOG = LogFactory private static final Log LOG = LogFactory
.getLog(ApplicationHistoryClientService.class); .getLog(ApplicationHistoryClientService.class);
@ -84,6 +86,12 @@ public class ApplicationHistoryClientService extends AbstractService {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
Preconditions.checkArgument(conf.getInt(
YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT) > 0,
"%s property value should be greater than zero",
YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT);
server = server =
rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler, rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler,
address, conf, null, conf.getInt( address, conf, null, conf.getInt(

View File

@ -76,6 +76,7 @@ import org.iq80.leveldb.WriteBatch;
import org.iq80.leveldb.WriteOptions; import org.iq80.leveldb.WriteOptions;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/** /**
* <p>An implementation of an application timeline store backed by leveldb.</p> * <p>An implementation of an application timeline store backed by leveldb.</p>
@ -194,6 +195,32 @@ public class LeveldbTimelineStore extends AbstractService
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected void serviceInit(Configuration conf) throws Exception { protected void serviceInit(Configuration conf) throws Exception {
Preconditions.checkArgument(conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_TTL_MS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_TTL_MS) > 0,
"%s property value should be greater than zero",
YarnConfiguration.TIMELINE_SERVICE_TTL_MS);
Preconditions.checkArgument(conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS) > 0,
"%s property value should be greater than zero",
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS);
Preconditions.checkArgument(conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE) >= 0,
"%s property value should be greater than or equal to zero",
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE);
Preconditions.checkArgument(conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE) > 0,
" %s property value should be greater than zero",
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE);
Preconditions.checkArgument(conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE) > 0,
"%s property value should be greater than zero",
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE);
Options options = new Options(); Options options = new Options();
options.createIfMissing(true); options.createIfMissing(true);
options.cacheSize(conf.getLong( options.cacheSize(conf.getLong(

View File

@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
import org.junit.After; import org.junit.After;
import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import java.util.HashMap; import java.util.HashMap;
@ -49,6 +50,19 @@ public class TestApplicationHistoryServer {
MemoryTimelineStore.class, TimelineStore.class); MemoryTimelineStore.class, TimelineStore.class);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0"); config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0");
try { try {
try {
historyServer.init(config);
config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
0);
historyServer.start();
fail();
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT));
}
config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT);
historyServer = new ApplicationHistoryServer();
historyServer.init(config); historyServer.init(config);
assertEquals(STATE.INITED, historyServer.getServiceState()); assertEquals(STATE.INITED, historyServer.getServiceState());
assertEquals(5, historyServer.getServices().size()); assertEquals(5, historyServer.getServices().size());

View File

@ -311,7 +311,77 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
e.getMessage().contains("Incompatible version for timeline store")); e.getMessage().contains("Incompatible version for timeline store"));
} }
} }
@Test
public void testValidateConfig() throws IOException {
Configuration copyConfig = new YarnConfiguration(config);
try {
Configuration newConfig = new YarnConfiguration(copyConfig);
newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0);
config = newConfig;
restartTimelineStore();
Assert.fail();
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
}
try {
Configuration newConfig = new YarnConfiguration(copyConfig);
newConfig.setLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0);
config = newConfig;
restartTimelineStore();
Assert.fail();
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
}
try {
Configuration newConfig = new YarnConfiguration(copyConfig);
newConfig.setLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1);
config = newConfig;
restartTimelineStore();
Assert.fail();
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
}
try {
Configuration newConfig = new YarnConfiguration(copyConfig);
newConfig
.setLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE,
0);
config = newConfig;
restartTimelineStore();
Assert.fail();
} catch (IllegalArgumentException e) {
Assert
.assertTrue(e
.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
}
try {
Configuration newConfig = new YarnConfiguration(copyConfig);
newConfig
.setLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE,
0);
config = newConfig;
restartTimelineStore();
Assert.fail();
} catch (IllegalArgumentException e) {
Assert
.assertTrue(e
.getMessage()
.contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE));
}
config = copyConfig;
restartTimelineStore();
}
private void restartTimelineStore() throws IOException { private void restartTimelineStore() throws IOException {
// need to close so leveldb releases database lock // need to close so leveldb releases database lock
if (store != null) { if (store != null) {