Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4102. Add a "skip existing table" mode for timeline schema creator (Li Lu via sjlee)

This commit is contained in:
Sangjin Lee 2015-09-11 09:46:13 -07:00
parent 8d9476ec5f
commit ee081018e4
1 changed files with 66 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
View File

@ -54,6 +54,11 @@ public class TimelineSchemaCreator {
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
private static final String PHOENIX_OPTION_SHORT = "p";
private static final String SKIP_EXISTING_TABLE_OPTION_SHORT = "s";
private static final String APP_TABLE_NAME_SHORT = "a";
private static final String APP_TO_FLOW_TABLE_NAME_SHORT = "a2f";
private static final String TTL_OPTION_SHORT = "m";
private static final String ENTITY_TABLE_NAME_SHORT = "e";
public static void main(String[] args) throws Exception {
@ -66,22 +71,25 @@ public class TimelineSchemaCreator {
CommandLine commandLine = parseArgs(otherArgs);
// Grab the entityTableName argument
String entityTableName = commandLine.getOptionValue("e");
String entityTableName
= commandLine.getOptionValue(ENTITY_TABLE_NAME_SHORT);
if (StringUtils.isNotBlank(entityTableName)) {
hbaseConf.set(EntityTable.TABLE_NAME_CONF_NAME, entityTableName);
}
String entityTableTTLMetrics = commandLine.getOptionValue("m");
String entityTableTTLMetrics = commandLine.getOptionValue(TTL_OPTION_SHORT);
if (StringUtils.isNotBlank(entityTableTTLMetrics)) {
int metricsTTL = Integer.parseInt(entityTableTTLMetrics);
new EntityTable().setMetricsTTL(metricsTTL, hbaseConf);
}
// Grab the appToflowTableName argument
String appToflowTableName = commandLine.getOptionValue("a2f");
String appToflowTableName = commandLine.getOptionValue(
APP_TO_FLOW_TABLE_NAME_SHORT);
if (StringUtils.isNotBlank(appToflowTableName)) {
hbaseConf.set(AppToFlowTable.TABLE_NAME_CONF_NAME, appToflowTableName);
}
// Grab the applicationTableName argument
String applicationTableName = commandLine.getOptionValue("a");
String applicationTableName = commandLine.getOptionValue(
APP_TABLE_NAME_SHORT);
if (StringUtils.isNotBlank(applicationTableName)) {
hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
applicationTableName);
@ -89,7 +97,13 @@ public class TimelineSchemaCreator {
List<Exception> exceptions = new ArrayList<>();
try {
createAllTables(hbaseConf);
boolean skipExisting
= commandLine.hasOption(SKIP_EXISTING_TABLE_OPTION_SHORT);
if (skipExisting) {
LOG.info("Will skip existing tables and continue on htable creation "
+ "exceptions!");
}
createAllTables(hbaseConf, skipExisting);
LOG.info("Successfully created HBase schema. ");
} catch (IOException e) {
LOG.error("Error in creating hbase tables: " + e.getMessage());
@ -135,26 +149,39 @@ public class TimelineSchemaCreator {
Options options = new Options();
// Input
Option o = new Option("e", "entityTableName", true, "entity table name");
Option o = new Option(ENTITY_TABLE_NAME_SHORT, "entityTableName", true,
"entity table name");
o.setArgName("entityTableName");
o.setRequired(false);
options.addOption(o);
o = new Option("m", "metricsTTL", true, "TTL for metrics column family");
o = new Option(TTL_OPTION_SHORT, "metricsTTL", true,
"TTL for metrics column family");
o.setArgName("metricsTTL");
o.setRequired(false);
options.addOption(o);
o = new Option("a2f", "appToflowTableName", true, "app to flow table name");
o = new Option(APP_TO_FLOW_TABLE_NAME_SHORT, "appToflowTableName", true,
"app to flow table name");
o.setArgName("appToflowTableName");
o = new Option("a", "applicationTableName", true, "application table name");
o.setRequired(false);
options.addOption(o);
o = new Option(APP_TABLE_NAME_SHORT, "applicationTableName", true,
"application table name");
o.setArgName("applicationTableName");
o.setRequired(false);
options.addOption(o);
// Options without an argument
// No need to set arg name since we do not need an argument here
o = new Option(PHOENIX_OPTION_SHORT, "usePhoenix", false,
"create Phoenix offline aggregation tables");
// No need to set arg name since we do not need an argument here
o.setRequired(false);
options.addOption(o);
o = new Option(SKIP_EXISTING_TABLE_OPTION_SHORT, "skipExistingTable",
false, "skip existing Hbase tables and continue to create new tables");
o.setRequired(false);
options.addOption(o);
@ -172,8 +199,8 @@ public class TimelineSchemaCreator {
return commandLine;
}
private static void createAllTables(Configuration hbaseConf)
throws IOException {
private static void createAllTables(Configuration hbaseConf,
boolean skipExisting) throws IOException {
Connection conn = null;
try {
@ -182,9 +209,33 @@ public class TimelineSchemaCreator {
if (admin == null) {
throw new IOException("Cannot create table since admin is null");
}
new EntityTable().createTable(admin, hbaseConf);
new AppToFlowTable().createTable(admin, hbaseConf);
new ApplicationTable().createTable(admin, hbaseConf);
try {
new EntityTable().createTable(admin, hbaseConf);
} catch (IOException e) {
if (skipExisting) {
LOG.warn("Skip and continue on: " + e.getMessage());
} else {
throw e;
}
}
try {
new AppToFlowTable().createTable(admin, hbaseConf);
} catch (IOException e) {
if (skipExisting) {
LOG.warn("Skip and continue on: " + e.getMessage());
} else {
throw e;
}
}
try {
new ApplicationTable().createTable(admin, hbaseConf);
} catch (IOException e) {
if (skipExisting) {
LOG.warn("Skip and continue on: " + e.getMessage());
} else {
throw e;
}
}
} finally {
if (conn != null) {
conn.close();