YARN-6853. Add MySql Scripts for FederationStateStore. (Contributed by Giovanni Matteo Fumarola via curino)
This commit is contained in:
Carlo Curino
parent
c581e94384
commit
874ddbf0b5
|
|
@ -0,0 +1,21 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to create a new Database in MySQL for the Federation StateStore
|
||||
|
||||
CREATE database FederationStateStore;
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to generate all the stored procedures for the Federation StateStore in MySQL
|
||||
|
||||
USE FederationStateStore
|
||||
|
||||
DELIMITER //
|
||||
|
||||
CREATE PROCEDURE sp_registerSubCluster(
|
||||
IN subClusterId_IN varchar(256),
|
||||
IN amRMServiceAddress_IN varchar(256),
|
||||
IN clientRMServiceAddress_IN varchar(256),
|
||||
IN rmAdminServiceAddress_IN varchar(256),
|
||||
IN rmWebServiceAddress_IN varchar(256),
|
||||
IN state_IN varchar(256),
|
||||
IN lastStartTime_IN bigint, IN capability_IN varchar(6000),
|
||||
OUT rowCount_OUT int)
|
||||
BEGIN
|
||||
DELETE FROM membership WHERE (subClusterId = subClusterId_IN);
|
||||
INSERT INTO membership (subClusterId, amRMServiceAddress, clientRMServiceAddress,
|
||||
rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat, state, lastStartTime, capability)
|
||||
VALUES (subClusterId_IN, amRMServiceAddress_IN, clientRMServiceAddress_IN,
|
||||
rmAdminServiceAddress_IN, rmWebServiceAddress_IN, NOW(), state_IN, lastStartTime_IN, capability_IN);
|
||||
SELECT ROW_COUNT() INTO rowCount_OUT;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_deregisterSubCluster(
|
||||
IN subClusterId_IN varchar(256),
|
||||
IN state_IN varchar(64),
|
||||
OUT rowCount_OUT int)
|
||||
BEGIN
|
||||
UPDATE membership SET state = state_IN
|
||||
WHERE (subClusterId = subClusterId_IN AND state != state_IN);
|
||||
SELECT ROW_COUNT() INTO rowCount_OUT;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_subClusterHeartbeat(
|
||||
IN subClusterId_IN varchar(256), IN state_IN varchar(64),
|
||||
IN capability_IN varchar(6000), OUT rowCount_OUT int)
|
||||
BEGIN
|
||||
UPDATE membership
|
||||
SET capability = capability_IN,
|
||||
state = state_IN,
|
||||
lastHeartBeat = NOW()
|
||||
WHERE subClusterId = subClusterId_IN;
|
||||
SELECT ROW_COUNT() INTO rowCount_OUT;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_getSubCluster(
|
||||
IN subClusterId_IN varchar(256),
|
||||
OUT amRMServiceAddress_OUT varchar(256),
|
||||
OUT clientRMServiceAddress_OUT varchar(256),
|
||||
OUT rmAdminServiceAddress_OUT varchar(256),
|
||||
OUT rmWebServiceAddress_OUT varchar(256),
|
||||
OUT lastHeartBeat_OUT datetime, OUT state_OUT varchar(64),
|
||||
OUT lastStartTime_OUT bigint,
|
||||
OUT capability_OUT varchar(6000))
|
||||
BEGIN
|
||||
SELECT amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
|
||||
lastHeartBeat, state, lastStartTime, capability
|
||||
INTO amRMServiceAddress_OUT, clientRMServiceAddress_OUT, rmAdminServiceAddress_OUT,
|
||||
rmWebServiceAddress_OUT, lastHeartBeat_OUT, state_OUT, lastStartTime_OUT, capability_OUT
|
||||
FROM membership WHERE subClusterId = subClusterId_IN;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_getSubClusters()
|
||||
BEGIN
|
||||
SELECT subClusterId, amRMServiceAddress, clientRMServiceAddress,
|
||||
rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat,
|
||||
state, lastStartTime, capability
|
||||
FROM membership;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_addApplicationHomeSubCluster(
|
||||
IN applicationId_IN varchar(64), IN homeSubCluster_IN varchar(256),
|
||||
OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)
|
||||
BEGIN
|
||||
INSERT INTO applicationsHomeSubCluster
|
||||
(applicationId,homeSubCluster)
|
||||
(SELECT applicationId_IN, homeSubCluster_IN
|
||||
FROM applicationsHomeSubCluster
|
||||
WHERE applicationId = applicationId_IN
|
||||
HAVING COUNT(*) = 0 );
|
||||
SELECT ROW_COUNT() INTO rowCount_OUT;
|
||||
SELECT homeSubCluster INTO storedHomeSubCluster_OUT
|
||||
FROM applicationsHomeSubCluster
|
||||
WHERE applicationId = applicationID_IN;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_updateApplicationHomeSubCluster(
|
||||
IN applicationId_IN varchar(64),
|
||||
IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)
|
||||
BEGIN
|
||||
UPDATE applicationsHomeSubCluster
|
||||
SET homeSubCluster = homeSubCluster_IN
|
||||
WHERE applicationId = applicationId_IN;
|
||||
SELECT ROW_COUNT() INTO rowCount_OUT;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_getApplicationHomeSubCluster(
|
||||
IN applicationId_IN varchar(64),
|
||||
OUT homeSubCluster_OUT varchar(256))
|
||||
BEGIN
|
||||
SELECT homeSubCluster INTO homeSubCluster_OUT
|
||||
FROM applicationsHomeSubCluster
|
||||
WHERE applicationId = applicationID_IN;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_getApplicationsHomeSubCluster()
|
||||
BEGIN
|
||||
SELECT applicationId, homeSubCluster
|
||||
FROM applicationsHomeSubCluster;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_deleteApplicationHomeSubCluster(
|
||||
IN applicationId_IN varchar(64), OUT rowCount_OUT int)
|
||||
BEGIN
|
||||
DELETE FROM applicationsHomeSubCluster
|
||||
WHERE applicationId = applicationId_IN;
|
||||
SELECT ROW_COUNT() INTO rowCount_OUT;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_setPolicyConfiguration(
|
||||
IN queue_IN varchar(256), IN policyType_IN varchar(256),
|
||||
IN params_IN varbinary(32768), OUT rowCount_OUT int)
|
||||
BEGIN
|
||||
DELETE FROM policies WHERE queue = queue_IN;
|
||||
INSERT INTO policies (queue, policyType, params)
|
||||
VALUES (queue_IN, policyType_IN, params_IN);
|
||||
SELECT ROW_COUNT() INTO rowCount_OUT;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_getPoliciesConfigurations()
|
||||
BEGIN
|
||||
SELECT queue, policyType, params FROM policies;
|
||||
END //
|
||||
|
||||
CREATE PROCEDURE sp_getPolicyConfiguration(
|
||||
IN queue_IN varchar(256), OUT policyType_OUT varchar(256),
|
||||
OUT params_OUT varbinary(32768))
|
||||
BEGIN
|
||||
SELECT policyType, params INTO policyType_OUT, params_OUT
|
||||
FROM policies WHERE queue = queue_IN;
|
||||
END //
|
||||
|
||||
DELIMITER ;
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to generate all the tables for the Federation StateStore in MySQL
|
||||
|
||||
USE FederationStateStore
|
||||
|
||||
CREATE TABLE applicationsHomeSubCluster(
|
||||
applicationId varchar(64) NOT NULL,
|
||||
subClusterId varchar(256) NULL,
|
||||
CONSTRAINT pk_applicationId PRIMARY KEY (applicationId)
|
||||
);
|
||||
|
||||
CREATE TABLE membership(
|
||||
subClusterId varchar(256) NOT NULL,
|
||||
amRMServiceAddress varchar(256) NOT NULL,
|
||||
clientRMServiceAddress varchar(256) NOT NULL,
|
||||
rmAdminServiceAddress varchar(256) NOT NULL,
|
||||
rmWebServiceAddress varchar(256) NOT NULL,
|
||||
lastHeartBeat datetime NOT NULL,
|
||||
state varchar(32) NOT NULL,
|
||||
lastStartTime bigint NULL,
|
||||
capability varchar(6000),
|
||||
CONSTRAINT pk_subClusterId PRIMARY KEY (subClusterId)
|
||||
);
|
||||
|
||||
CREATE TABLE policies(
|
||||
queue varchar(256) NOT NULL,
|
||||
policyType varchar(256) NOT NULL,
|
||||
params varbinary(32768),
|
||||
CONSTRAINT pk_queue PRIMARY KEY (queue)
|
||||
);
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to create a new User in MySQL for the Federation StateStore
|
||||
|
||||
CREATE USER 'FederationUser'@'%' IDENTIFIED BY 'FederationPassword';
|
||||
|
||||
GRANT ALL PRIVILEGES ON FederationStateStore.* TO 'FederationUser'@'%';
|
||||
|
||||
FLUSH PRIVILEGES;
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to drop the Federation StateStore in MySQL
|
||||
|
||||
DROP DATABASE FederationStateStore;
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to drop all the stored procedures for the Federation StateStore in MySQL
|
||||
|
||||
USE FederationStateStore
|
||||
|
||||
DROP PROCEDURE sp_registerSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_deregisterSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_subClusterHeartbeat;
|
||||
|
||||
DROP PROCEDURE sp_getSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_getSubClusters;
|
||||
|
||||
DROP PROCEDURE sp_addApplicationHomeSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_updateApplicationHomeSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_getApplicationHomeSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_getApplicationsHomeSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_deleteApplicationHomeSubCluster;
|
||||
|
||||
DROP PROCEDURE sp_setPolicyConfiguration;
|
||||
|
||||
DROP PROCEDURE sp_getPolicyConfiguration;
|
||||
|
||||
DROP PROCEDURE sp_getPoliciesConfigurations;
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to drop all the tables from the Federation StateStore in MySQL
|
||||
|
||||
USE FederationStateStore
|
||||
|
||||
DROP TABLE applicationsHomeSubCluster;
|
||||
|
||||
DROP TABLE membership;
|
||||
|
||||
DROP TABLE policies;
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Script to drop the user from Federation StateStore in MySQL
|
||||
|
||||
DROP USER 'FederationUser'@'%';
|
||||
|
|
@ -161,8 +161,8 @@ These are common configurations that should appear in the **conf/yarn-site.xml**
|
|||
|:---- |:---- |
|
||||
|`yarn.federation.enabled` | `true` | Whether federation is enabled or not |
|
||||
|`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. |
|
||||
|`yarn.federation.state-store.sql.url` | `jdbc:sqlserver://<host>:<port>;databaseName=FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
|
||||
|`yarn.federation.state-store.sql.jdbc-class` | `com.microsoft.sqlserver.jdbc.SQLServerDataSource` | For SQLFederationStateStore the jdbc class to use. |
|
||||
|`yarn.federation.state-store.sql.url` | `jdbc:mysql://<host>:<port>/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
|
||||
|`yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. |
|
||||
|`yarn.federation.state-store.sql.username` | `<dbuser>` | For SQLFederationStateStore the username for the DB connection. |
|
||||
|`yarn.federation.state-store.sql.password` | `<dbpass>` | For SQLFederationStateStore the password for the DB connection. |
|
||||
|`yarn.resourcemanager.cluster-id` | `<unique-subcluster-id>` | The unique subcluster identifier for this RM (same as the one used for HA). |
|
||||
|
|
@ -238,7 +238,19 @@ Optional:
|
|||
|
||||
###State-Store:
|
||||
|
||||
Currently, the only supported implementation of the state-store is Microsoft SQL Server. After [setting up](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) such an instance of SQL Server, set up the database for use by the federation system. This can be done by running the following SQL files in the database: **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql** and **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreTables.sql**
|
||||
Currently, we support only SQL based implementation of state-store (ZooKeeper is in the works), i.e. either MySQL or Microsoft SQL Server.
|
||||
|
||||
For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH.
|
||||
Then the DB schema is created by executing the following SQL scripts in the database:
|
||||
1. **sbin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql**.
|
||||
2. **sbin/FederationStateStore/MySQL/FederationStateStoreUser.sql**.
|
||||
3. **sbin/FederationStateStore/MySQL/FederationStateStoreTables.sql**.
|
||||
4. **sbin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql**.
|
||||
In the same directory we provide scripts to drop the Stored Procedures, the Tables, the User and the Database.
|
||||
**Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password.
|
||||
|
||||
For SQL-Server, the process is similar, but the jdbc driver is already included in the pom (license allows it).
|
||||
SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
|
||||
|
||||
Running a Sample Job
|
||||
--------------------
|
||||
|
|
|
|||
Loading…
Reference in New Issue