From 874ddbf0b5b1d34aca70ee7fc303cbffdde67236 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Tue, 1 Aug 2017 17:18:20 -0700 Subject: [PATCH] YARN-6853. Add MySql Scripts for FederationStateStore. (Contributed by Giovanni Matteo Fumarola via curino) --- .../MySQL/FederationStateStoreDatabase.sql | 21 +++ .../MySQL/FederationStateStoreStoredProcs.sql | 162 ++++++++++++++++++ .../MySQL/FederationStateStoreTables.sql | 47 +++++ .../MySQL/FederationStateStoreUser.sql | 25 +++ .../MySQL/dropDatabase.sql | 21 +++ .../MySQL/dropStoreProcedures.sql | 47 +++++ .../FederationStateStore/MySQL/dropTables.sql | 27 +++ .../FederationStateStore/MySQL/dropUser.sql | 21 +++ .../src/site/markdown/Federation.md | 18 +- 9 files changed, 386 insertions(+), 3 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreUser.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropDatabase.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropUser.sql diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql new file mode 100644 index 00000000000..68649e6300d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to create a new Database in MySQL for the Federation StateStore + +CREATE database FederationStateStore; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql new file mode 100644 index 00000000000..eae882e4a48 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to generate all the stored procedures for the Federation StateStore in MySQL + +USE FederationStateStore + +DELIMITER // + +CREATE PROCEDURE sp_registerSubCluster( + IN subClusterId_IN varchar(256), + IN amRMServiceAddress_IN varchar(256), + IN clientRMServiceAddress_IN varchar(256), + IN rmAdminServiceAddress_IN varchar(256), + IN rmWebServiceAddress_IN varchar(256), + IN state_IN varchar(256), + IN lastStartTime_IN bigint, IN capability_IN varchar(6000), + OUT rowCount_OUT int) +BEGIN + DELETE FROM membership WHERE (subClusterId = subClusterId_IN); + INSERT INTO membership (subClusterId, amRMServiceAddress, clientRMServiceAddress, + rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat, state, lastStartTime, capability) + VALUES (subClusterId_IN, amRMServiceAddress_IN, clientRMServiceAddress_IN, + rmAdminServiceAddress_IN, rmWebServiceAddress_IN, NOW(), state_IN, lastStartTime_IN, capability_IN); + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + +CREATE PROCEDURE sp_deregisterSubCluster( + IN subClusterId_IN varchar(256), + IN state_IN varchar(64), + OUT rowCount_OUT int) +BEGIN + UPDATE membership SET state = state_IN + WHERE (subClusterId = subClusterId_IN AND state != state_IN); + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + +CREATE PROCEDURE sp_subClusterHeartbeat( + IN subClusterId_IN varchar(256), IN state_IN varchar(64), + IN capability_IN varchar(6000), OUT rowCount_OUT int) +BEGIN + UPDATE membership + SET capability = capability_IN, + state = state_IN, + lastHeartBeat = NOW() + WHERE subClusterId = subClusterId_IN; + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + +CREATE PROCEDURE sp_getSubCluster( + IN subClusterId_IN varchar(256), + OUT amRMServiceAddress_OUT varchar(256), + OUT clientRMServiceAddress_OUT varchar(256), + OUT rmAdminServiceAddress_OUT varchar(256), + OUT rmWebServiceAddress_OUT varchar(256), + OUT lastHeartBeat_OUT datetime, OUT state_OUT varchar(64), + OUT lastStartTime_OUT bigint, + OUT capability_OUT varchar(6000)) +BEGIN + SELECT amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress, + lastHeartBeat, state, lastStartTime, capability + INTO amRMServiceAddress_OUT, clientRMServiceAddress_OUT, rmAdminServiceAddress_OUT, + rmWebServiceAddress_OUT, lastHeartBeat_OUT, state_OUT, lastStartTime_OUT, capability_OUT + FROM membership WHERE subClusterId = subClusterId_IN; +END // + +CREATE PROCEDURE sp_getSubClusters() +BEGIN + SELECT subClusterId, amRMServiceAddress, clientRMServiceAddress, + rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat, + state, lastStartTime, capability + FROM membership; +END // + +CREATE PROCEDURE sp_addApplicationHomeSubCluster( + IN applicationId_IN varchar(64), IN homeSubCluster_IN varchar(256), + OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int) +BEGIN + INSERT INTO applicationsHomeSubCluster + (applicationId,homeSubCluster) + (SELECT applicationId_IN, homeSubCluster_IN + FROM applicationsHomeSubCluster + WHERE applicationId = applicationId_IN + HAVING COUNT(*) = 0 ); + SELECT ROW_COUNT() INTO rowCount_OUT; + SELECT homeSubCluster INTO storedHomeSubCluster_OUT + FROM applicationsHomeSubCluster + WHERE applicationId = applicationID_IN; +END // + +CREATE PROCEDURE sp_updateApplicationHomeSubCluster( + IN applicationId_IN varchar(64), + IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int) +BEGIN + UPDATE applicationsHomeSubCluster + SET homeSubCluster = homeSubCluster_IN + WHERE applicationId = applicationId_IN; + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + +CREATE PROCEDURE sp_getApplicationHomeSubCluster( + IN applicationId_IN varchar(64), + OUT homeSubCluster_OUT varchar(256)) +BEGIN + SELECT homeSubCluster INTO homeSubCluster_OUT + FROM applicationsHomeSubCluster + WHERE applicationId = applicationID_IN; +END // + +CREATE PROCEDURE sp_getApplicationsHomeSubCluster() +BEGIN + SELECT applicationId, homeSubCluster + FROM applicationsHomeSubCluster; +END // + +CREATE PROCEDURE sp_deleteApplicationHomeSubCluster( + IN applicationId_IN varchar(64), OUT rowCount_OUT int) +BEGIN + DELETE FROM applicationsHomeSubCluster + WHERE applicationId = applicationId_IN; + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + +CREATE PROCEDURE sp_setPolicyConfiguration( + IN queue_IN varchar(256), IN policyType_IN varchar(256), + IN params_IN varbinary(32768), OUT rowCount_OUT int) +BEGIN + DELETE FROM policies WHERE queue = queue_IN; + INSERT INTO policies (queue, policyType, params) + VALUES (queue_IN, policyType_IN, params_IN); + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + +CREATE PROCEDURE sp_getPoliciesConfigurations() +BEGIN + SELECT queue, policyType, params FROM policies; +END // + +CREATE PROCEDURE sp_getPolicyConfiguration( + IN queue_IN varchar(256), OUT policyType_OUT varchar(256), + OUT params_OUT varbinary(32768)) +BEGIN + SELECT policyType, params INTO policyType_OUT, params_OUT + FROM policies WHERE queue = queue_IN; +END // + +DELIMITER ; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql new file mode 100644 index 00000000000..67a181754b9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to generate all the tables for the Federation StateStore in MySQL + +USE FederationStateStore + +CREATE TABLE applicationsHomeSubCluster( + applicationId varchar(64) NOT NULL, + subClusterId varchar(256) NULL, + CONSTRAINT pk_applicationId PRIMARY KEY (applicationId) +); + +CREATE TABLE membership( + subClusterId varchar(256) NOT NULL, + amRMServiceAddress varchar(256) NOT NULL, + clientRMServiceAddress varchar(256) NOT NULL, + rmAdminServiceAddress varchar(256) NOT NULL, + rmWebServiceAddress varchar(256) NOT NULL, + lastHeartBeat datetime NOT NULL, + state varchar(32) NOT NULL, + lastStartTime bigint NULL, + capability varchar(6000), + CONSTRAINT pk_subClusterId PRIMARY KEY (subClusterId) +); + +CREATE TABLE policies( + queue varchar(256) NOT NULL, + policyType varchar(256) NOT NULL, + params varbinary(32768), + CONSTRAINT pk_queue PRIMARY KEY (queue) +); diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreUser.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreUser.sql new file mode 100644 index 00000000000..32f49334656 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreUser.sql @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to create a new User in MySQL for the Federation StateStore + +CREATE USER 'FederationUser'@'%' IDENTIFIED BY 'FederationPassword'; + +GRANT ALL PRIVILEGES ON FederationStateStore.* TO 'FederationUser'@'%'; + +FLUSH PRIVILEGES; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropDatabase.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropDatabase.sql new file mode 100644 index 00000000000..c915bfe510a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropDatabase.sql @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop the Federation StateStore in MySQL + +DROP DATABASE FederationStateStore; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql new file mode 100644 index 00000000000..f24f3fb22b5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop all the stored procedures for the Federation StateStore in MySQL + +USE FederationStateStore + +DROP PROCEDURE sp_registerSubCluster; + +DROP PROCEDURE sp_deregisterSubCluster; + +DROP PROCEDURE sp_subClusterHeartbeat; + +DROP PROCEDURE sp_getSubCluster; + +DROP PROCEDURE sp_getSubClusters; + +DROP PROCEDURE sp_addApplicationHomeSubCluster; + +DROP PROCEDURE sp_updateApplicationHomeSubCluster; + +DROP PROCEDURE sp_getApplicationHomeSubCluster; + +DROP PROCEDURE sp_getApplicationsHomeSubCluster; + +DROP PROCEDURE sp_deleteApplicationHomeSubCluster; + +DROP PROCEDURE sp_setPolicyConfiguration; + +DROP PROCEDURE sp_getPolicyConfiguration; + +DROP PROCEDURE sp_getPoliciesConfigurations; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql new file mode 100644 index 00000000000..ea6567b028b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop all the tables from the Federation StateStore in MySQL + +USE FederationStateStore + +DROP TABLE applicationsHomeSubCluster; + +DROP TABLE membership; + +DROP TABLE policies; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropUser.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropUser.sql new file mode 100644 index 00000000000..7b4bb02fb59 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropUser.sql @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop the user from Federation StateStore in MySQL + +DROP USER 'FederationUser'@'%'; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 61cb77f3d76..ecf61c5d417 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -161,8 +161,8 @@ These are common configurations that should appear in the **conf/yarn-site.xml** |:---- |:---- | |`yarn.federation.enabled` | `true` | Whether federation is enabled or not | |`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. | -|`yarn.federation.state-store.sql.url` | `jdbc:sqlserver://:;databaseName=FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. | -|`yarn.federation.state-store.sql.jdbc-class` | `com.microsoft.sqlserver.jdbc.SQLServerDataSource` | For SQLFederationStateStore the jdbc class to use. | +|`yarn.federation.state-store.sql.url` | `jdbc:mysql://:/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. | +|`yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. | |`yarn.federation.state-store.sql.username` | `` | For SQLFederationStateStore the username for the DB connection. | |`yarn.federation.state-store.sql.password` | `` | For SQLFederationStateStore the password for the DB connection. | |`yarn.resourcemanager.cluster-id` | `` | The unique subcluster identifier for this RM (same as the one used for HA). | @@ -238,7 +238,19 @@ Optional: ###State-Store: -Currently, the only supported implementation of the state-store is Microsoft SQL Server. After [setting up](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) such an instance of SQL Server, set up the database for use by the federation system. This can be done by running the following SQL files in the database: **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql** and **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreTables.sql** +Currently, we support only SQL based implementation of state-store (ZooKeeper is in the works), i.e. either MySQL or Microsoft SQL Server. + +For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH. +Then the DB schema is created by executing the following SQL scripts in the database: +1. **sbin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql**. +2. **sbin/FederationStateStore/MySQL/FederationStateStoreUser.sql**. +3. **sbin/FederationStateStore/MySQL/FederationStateStoreTables.sql**. +4. **sbin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql**. +In the same directory we provide scripts to drop the Stored Procedures, the Tables, the User and the Database. +**Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password. + +For SQL-Server, the process is similar, but the jdbc driver is already included in the pom (license allows it). +SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**. Running a Sample Job --------------------