YARN-2165. Added the sanity check for the numeric configuration values of the timeline service. Contributed by Vasanth kumar RJ.
This commit is contained in:
parent
978736d486
commit
8f1454cc6d
|
@ -98,6 +98,9 @@ Release 2.7.0 - UNRELEASED
|
||||||
YARN-2404. Removed ApplicationAttemptState and ApplicationState class in
|
YARN-2404. Removed ApplicationAttemptState and ApplicationState class in
|
||||||
RMStateStore. (Tsuyoshi OZAWA via jianhe)
|
RMStateStore. (Tsuyoshi OZAWA via jianhe)
|
||||||
|
|
||||||
|
YARN-2165. Added the sanity check for the numeric configuration values of
|
||||||
|
the timeline service. (Vasanth kumar RJ via zjshen)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
BUG FIXES
|
BUG FIXES
|
||||||
|
|
|
@ -67,6 +67,7 @@ import org.codehaus.jackson.map.ObjectMapper;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
import com.sun.jersey.api.client.Client;
|
import com.sun.jersey.api.client.Client;
|
||||||
import com.sun.jersey.api.client.ClientHandlerException;
|
import com.sun.jersey.api.client.ClientHandlerException;
|
||||||
import com.sun.jersey.api.client.ClientRequest;
|
import com.sun.jersey.api.client.ClientRequest;
|
||||||
|
@ -141,6 +142,18 @@ public class TimelineClientImpl extends TimelineClient {
|
||||||
|
|
||||||
// Constructor with default retry settings
|
// Constructor with default retry settings
|
||||||
public TimelineClientConnectionRetry(Configuration conf) {
|
public TimelineClientConnectionRetry(Configuration conf) {
|
||||||
|
Preconditions.checkArgument(conf.getInt(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES) >= -1,
|
||||||
|
"%s property value should be greater than or equal to -1",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
|
||||||
|
Preconditions
|
||||||
|
.checkArgument(
|
||||||
|
conf.getLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS) > 0,
|
||||||
|
"%s property value should be greater than zero",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS);
|
||||||
maxRetries = conf.getInt(
|
maxRetries = conf.getInt(
|
||||||
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
|
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
|
||||||
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
|
||||||
|
|
|
@ -1355,7 +1355,8 @@
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<description>
|
<description>
|
||||||
Default maximum number of retires for timeline servive client.
|
Default maximum number of retires for timeline servive client
|
||||||
|
and value -1 means no limit.
|
||||||
</description>
|
</description>
|
||||||
<name>yarn.timeline-service.client.max-retries</name>
|
<name>yarn.timeline-service.client.max-retries</name>
|
||||||
<value>30</value>
|
<value>30</value>
|
||||||
|
|
|
@ -154,6 +154,29 @@ public class TestTimelineClient {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testCheckRetryCount() throws Exception {
|
public void testCheckRetryCount() throws Exception {
|
||||||
|
try {
|
||||||
|
YarnConfiguration conf = new YarnConfiguration();
|
||||||
|
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
||||||
|
conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
|
||||||
|
-2);
|
||||||
|
createTimelineClient(conf);
|
||||||
|
Assert.fail();
|
||||||
|
} catch(IllegalArgumentException e) {
|
||||||
|
Assert.assertTrue(e.getMessage().contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES));
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
YarnConfiguration conf = new YarnConfiguration();
|
||||||
|
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
||||||
|
conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
|
||||||
|
0);
|
||||||
|
createTimelineClient(conf);
|
||||||
|
Assert.fail();
|
||||||
|
} catch(IllegalArgumentException e) {
|
||||||
|
Assert.assertTrue(e.getMessage().contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS));
|
||||||
|
}
|
||||||
int newMaxRetries = 5;
|
int newMaxRetries = 5;
|
||||||
long newIntervalMs = 500;
|
long newIntervalMs = 500;
|
||||||
YarnConfiguration conf = new YarnConfiguration();
|
YarnConfiguration conf = new YarnConfiguration();
|
||||||
|
|
|
@ -61,6 +61,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider;
|
import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
public class ApplicationHistoryClientService extends AbstractService {
|
public class ApplicationHistoryClientService extends AbstractService {
|
||||||
private static final Log LOG = LogFactory
|
private static final Log LOG = LogFactory
|
||||||
.getLog(ApplicationHistoryClientService.class);
|
.getLog(ApplicationHistoryClientService.class);
|
||||||
|
@ -84,6 +86,12 @@ public class ApplicationHistoryClientService extends AbstractService {
|
||||||
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
|
||||||
|
|
||||||
|
Preconditions.checkArgument(conf.getInt(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT) > 0,
|
||||||
|
"%s property value should be greater than zero",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT);
|
||||||
|
|
||||||
server =
|
server =
|
||||||
rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler,
|
rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler,
|
||||||
address, conf, null, conf.getInt(
|
address, conf, null, conf.getInt(
|
||||||
|
|
|
@ -76,6 +76,7 @@ import org.iq80.leveldb.WriteBatch;
|
||||||
import org.iq80.leveldb.WriteOptions;
|
import org.iq80.leveldb.WriteOptions;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>An implementation of an application timeline store backed by leveldb.</p>
|
* <p>An implementation of an application timeline store backed by leveldb.</p>
|
||||||
|
@ -194,6 +195,32 @@ public class LeveldbTimelineStore extends AbstractService
|
||||||
@Override
|
@Override
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
protected void serviceInit(Configuration conf) throws Exception {
|
protected void serviceInit(Configuration conf) throws Exception {
|
||||||
|
Preconditions.checkArgument(conf.getLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_TTL_MS,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_TTL_MS) > 0,
|
||||||
|
"%s property value should be greater than zero",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_TTL_MS);
|
||||||
|
Preconditions.checkArgument(conf.getLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS) > 0,
|
||||||
|
"%s property value should be greater than zero",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS);
|
||||||
|
Preconditions.checkArgument(conf.getLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE) >= 0,
|
||||||
|
"%s property value should be greater than or equal to zero",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE);
|
||||||
|
Preconditions.checkArgument(conf.getLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE) > 0,
|
||||||
|
" %s property value should be greater than zero",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE);
|
||||||
|
Preconditions.checkArgument(conf.getLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE) > 0,
|
||||||
|
"%s property value should be greater than zero",
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE);
|
||||||
|
|
||||||
Options options = new Options();
|
Options options = new Options();
|
||||||
options.createIfMissing(true);
|
options.createIfMissing(true);
|
||||||
options.cacheSize(conf.getLong(
|
options.cacheSize(conf.getLong(
|
||||||
|
|
|
@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
|
||||||
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
|
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
|
||||||
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
|
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
@ -49,6 +50,19 @@ public class TestApplicationHistoryServer {
|
||||||
MemoryTimelineStore.class, TimelineStore.class);
|
MemoryTimelineStore.class, TimelineStore.class);
|
||||||
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0");
|
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0");
|
||||||
try {
|
try {
|
||||||
|
try {
|
||||||
|
historyServer.init(config);
|
||||||
|
config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
|
||||||
|
0);
|
||||||
|
historyServer.start();
|
||||||
|
fail();
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
Assert.assertTrue(e.getMessage().contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT));
|
||||||
|
}
|
||||||
|
config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
|
||||||
|
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT);
|
||||||
|
historyServer = new ApplicationHistoryServer();
|
||||||
historyServer.init(config);
|
historyServer.init(config);
|
||||||
assertEquals(STATE.INITED, historyServer.getServiceState());
|
assertEquals(STATE.INITED, historyServer.getServiceState());
|
||||||
assertEquals(5, historyServer.getServices().size());
|
assertEquals(5, historyServer.getServices().size());
|
||||||
|
|
|
@ -311,7 +311,77 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
|
||||||
e.getMessage().contains("Incompatible version for timeline store"));
|
e.getMessage().contains("Incompatible version for timeline store"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testValidateConfig() throws IOException {
|
||||||
|
Configuration copyConfig = new YarnConfiguration(config);
|
||||||
|
try {
|
||||||
|
Configuration newConfig = new YarnConfiguration(copyConfig);
|
||||||
|
newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0);
|
||||||
|
config = newConfig;
|
||||||
|
restartTimelineStore();
|
||||||
|
Assert.fail();
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
Assert.assertTrue(e.getMessage().contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Configuration newConfig = new YarnConfiguration(copyConfig);
|
||||||
|
newConfig.setLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0);
|
||||||
|
config = newConfig;
|
||||||
|
restartTimelineStore();
|
||||||
|
Assert.fail();
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
Assert.assertTrue(e.getMessage().contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Configuration newConfig = new YarnConfiguration(copyConfig);
|
||||||
|
newConfig.setLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1);
|
||||||
|
config = newConfig;
|
||||||
|
restartTimelineStore();
|
||||||
|
Assert.fail();
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
Assert.assertTrue(e.getMessage().contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Configuration newConfig = new YarnConfiguration(copyConfig);
|
||||||
|
newConfig
|
||||||
|
.setLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE,
|
||||||
|
0);
|
||||||
|
config = newConfig;
|
||||||
|
restartTimelineStore();
|
||||||
|
Assert.fail();
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
Assert
|
||||||
|
.assertTrue(e
|
||||||
|
.getMessage().contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Configuration newConfig = new YarnConfiguration(copyConfig);
|
||||||
|
newConfig
|
||||||
|
.setLong(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE,
|
||||||
|
0);
|
||||||
|
config = newConfig;
|
||||||
|
restartTimelineStore();
|
||||||
|
Assert.fail();
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
Assert
|
||||||
|
.assertTrue(e
|
||||||
|
.getMessage()
|
||||||
|
.contains(
|
||||||
|
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE));
|
||||||
|
}
|
||||||
|
config = copyConfig;
|
||||||
|
restartTimelineStore();
|
||||||
|
}
|
||||||
|
|
||||||
private void restartTimelineStore() throws IOException {
|
private void restartTimelineStore() throws IOException {
|
||||||
// need to close so leveldb releases database lock
|
// need to close so leveldb releases database lock
|
||||||
if (store != null) {
|
if (store != null) {
|
||||||
|
|
Loading…
Reference in New Issue