From 79ac70ac86fb3c16ed70e3e50b6e4d241311c897 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Fri, 1 Dec 2017 12:50:04 -0800 Subject: [PATCH] HBASE-19407 [branch-2] Remove backup/restore --- bin/hbase | 6 - hbase-assembly/pom.xml | 4 - .../src/main/assembly/hadoop-two-compat.xml | 1 - hbase-backup/pom.xml | 291 --- .../hadoop/hbase/backup/BackupAdmin.java | 136 -- .../hbase/backup/BackupClientFactory.java | 56 - .../hadoop/hbase/backup/BackupCopyJob.java | 55 - .../hadoop/hbase/backup/BackupDriver.java | 214 -- .../hbase/backup/BackupHFileCleaner.java | 180 -- .../hadoop/hbase/backup/BackupInfo.java | 550 ----- .../hadoop/hbase/backup/BackupMergeJob.java | 40 - .../hadoop/hbase/backup/BackupObserver.java | 110 - .../hadoop/hbase/backup/BackupRequest.java | 153 -- .../hbase/backup/BackupRestoreConstants.java | 130 -- .../hbase/backup/BackupRestoreFactory.java | 82 - .../hadoop/hbase/backup/BackupTableInfo.java | 82 - .../hbase/backup/HBackupFileSystem.java | 146 -- .../apache/hadoop/hbase/backup/LogUtils.java | 50 - .../hadoop/hbase/backup/RestoreDriver.java | 275 --- .../hadoop/hbase/backup/RestoreJob.java | 46 - .../hadoop/hbase/backup/RestoreRequest.java | 135 -- .../hbase/backup/impl/BackupAdminImpl.java | 743 ------ .../hbase/backup/impl/BackupCommands.java | 1019 -------- .../hbase/backup/impl/BackupException.java | 84 - .../hbase/backup/impl/BackupManager.java | 502 ---- .../hbase/backup/impl/BackupManifest.java | 674 ------ .../hbase/backup/impl/BackupSystemTable.java | 2050 ----------------- .../backup/impl/FullTableBackupClient.java | 234 -- .../backup/impl/IncrementalBackupManager.java | 387 ---- .../impl/IncrementalTableBackupClient.java | 404 ---- .../backup/impl/RestoreTablesClient.java | 283 --- .../hbase/backup/impl/TableBackupClient.java | 436 ---- .../mapreduce/MapReduceBackupCopyJob.java | 438 ---- .../mapreduce/MapReduceBackupMergeJob.java | 321 --- .../mapreduce/MapReduceHFileSplitterJob.java | 172 -- .../backup/mapreduce/MapReduceRestoreJob.java | 136 -- .../hbase/backup/master/BackupLogCleaner.java | 143 -- .../master/LogRollMasterProcedureManager.java | 168 -- .../LogRollBackupSubprocedure.java | 167 -- .../LogRollBackupSubprocedurePool.java | 139 -- .../LogRollRegionServerProcedureManager.java | 181 -- .../hadoop/hbase/backup/util/BackupSet.java | 58 - .../hadoop/hbase/backup/util/BackupUtils.java | 747 ------ .../hadoop/hbase/backup/util/RestoreTool.java | 518 ----- .../hadoop/hbase/backup/TestBackupBase.java | 508 ---- .../hbase/backup/TestBackupBoundaryTests.java | 97 - .../backup/TestBackupCommandLineTool.java | 441 ---- .../hadoop/hbase/backup/TestBackupDelete.java | 102 - .../hbase/backup/TestBackupDeleteRestore.java | 70 - .../backup/TestBackupDeleteWithFailures.java | 199 -- .../hbase/backup/TestBackupDescribe.java | 110 - .../hbase/backup/TestBackupHFileCleaner.java | 141 -- .../backup/TestBackupMultipleDeletes.java | 158 -- .../hadoop/hbase/backup/TestBackupRepair.java | 91 - .../hbase/backup/TestBackupShowHistory.java | 148 -- .../backup/TestBackupStatusProgress.java | 96 - .../hbase/backup/TestBackupSystemTable.java | 511 ---- .../hadoop/hbase/backup/TestFullBackup.java | 59 - .../hbase/backup/TestFullBackupSet.java | 103 - .../backup/TestFullBackupSetRestoreSet.java | 128 - .../backup/TestFullBackupWithFailures.java | 79 - .../hadoop/hbase/backup/TestFullRestore.java | 345 --- .../hbase/backup/TestIncrementalBackup.java | 209 -- .../TestIncrementalBackupDeleteTable.java | 129 -- ...estIncrementalBackupMergeWithFailures.java | 336 --- .../TestIncrementalBackupWithBulkLoad.java | 144 -- .../TestIncrementalBackupWithFailures.java | 161 -- .../hadoop/hbase/backup/TestRemoteBackup.java | 135 -- .../hbase/backup/TestRemoteRestore.java | 59 - .../backup/TestRepairAfterFailedDelete.java | 93 - .../backup/TestRestoreBoundaryTests.java | 80 - .../hbase/backup/TestSystemTableSnapshot.java | 55 - .../backup/master/TestBackupLogCleaner.java | 162 -- .../src/test/resources/log4j.properties | 68 - hbase-it/pom.xml | 5 - .../hbase/IntegrationTestBackupRestore.java | 313 --- .../src/main/protobuf/Backup.proto | 117 - pom.xml | 6 - 78 files changed, 18204 deletions(-) delete mode 100644 hbase-backup/pom.xml delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupClientFactory.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyJob.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupMergeJob.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupObserver.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreFactory.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupTableInfo.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java delete mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java delete mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java delete mode 100644 hbase-backup/src/test/resources/log4j.properties delete mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java delete mode 100644 hbase-protocol-shaded/src/main/protobuf/Backup.proto diff --git a/bin/hbase b/bin/hbase index d98e7bc0c57..1fc51021035 100755 --- a/bin/hbase +++ b/bin/hbase @@ -103,8 +103,6 @@ if [ $# = 0 ]; then echo " ltt Run LoadTestTool" echo " canary Run the Canary tool" echo " version Print the version" - echo " backup Backup tables for recovery" - echo " restore Restore tables from existing backup image" echo " CLASSNAME Run the class named CLASSNAME" exit 1 fi @@ -363,10 +361,6 @@ elif [ "$COMMAND" = "hfile" ] ; then CLASS='org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter' elif [ "$COMMAND" = "zkcli" ] ; then CLASS="org.apache.hadoop.hbase.zookeeper.ZKMainServer" -elif [ "$COMMAND" = "backup" ] ; then - CLASS='org.apache.hadoop.hbase.backup.BackupDriver' -elif [ "$COMMAND" = "restore" ] ; then - CLASS='org.apache.hadoop.hbase.backup.RestoreDriver' elif [ "$COMMAND" = "upgrade" ] ; then echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0." echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading." diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index f5e535f7f7d..8e1d67a09a5 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -262,10 +262,6 @@ org.apache.httpcomponents httpcore - - org.apache.hbase - hbase-backup - org.apache.hbase hbase-replication diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index b9bac759430..3579476e75c 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -34,7 +34,6 @@ org.apache.hbase:hbase-annotations - org.apache.hbase:hbase-backup org.apache.hbase:hbase-client org.apache.hbase:hbase-common org.apache.hbase:hbase-examples diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml deleted file mode 100644 index 1b796241838..00000000000 --- a/hbase-backup/pom.xml +++ /dev/null @@ -1,291 +0,0 @@ - - - - 4.0.0 - - hbase-build-configuration - org.apache.hbase - 2.0.0-beta-1-SNAPSHOT - ../hbase-build-configuration - - hbase-backup - Apache HBase - Backup - Backup for HBase - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - - maven-assembly-plugin - - true - - - - - org.apache.maven.plugins - maven-source-plugin - - - - - - - org.eclipse.m2e - lifecycle-mapping - - - - - - - - - - - - - - org.apache.hbase - hbase-annotations - test-jar - test - - - org.apache.hbase - hbase-client - - - org.apache.hbase - hbase-server - - - org.apache.hbase - hbase-server - test-jar - test - - - org.apache.hbase - hbase-mapreduce - test-jar - test - - - org.apache.hbase - hbase-mapreduce - - - org.apache.hbase - hbase-common - - - org.apache.hbase - hbase-protocol-shaded - - - org.apache.hbase - hbase-testing-util - test - - - - commons-cli - commons-cli - - - org.apache.commons - commons-lang3 - - - commons-logging - commons-logging - - - org.apache.hbase.thirdparty - hbase-shaded-miscellaneous - - - org.apache.zookeeper - zookeeper - - - - log4j - log4j - - - junit - junit - test - - - - - - hadoop-2.0 - - - - - !hadoop.profile - - - - - org.apache.hadoop - hadoop-common - - - org.apache.htrace - htrace-core - - - net.java.dev.jets3t - jets3t - - - javax.servlet.jsp - jsp-api - - - org.mortbay.jetty - jetty - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - javax.servlet - servlet-api - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - com.google.code.findbugs - jsr305 - - - - - - - com.google.guava - guava - ${hadoop.guava.version} - - - com.google.code.findbugs - jsr305 - - - - - - io.netty - netty - ${netty.hadoop.version} - - - com.google.code.findbugs - jsr305 - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop-two.version} - - - javax.servlet - servlet-api - - - javax.inject - javax.inject - - - com.google.guava - guava - - - - - org.apache.hadoop - hadoop-distcp - ${hadoop-two.version} - - - - - - hadoop-3.0 - - - hadoop.profile - 3.0 - - - - 3.0-SNAPSHOT - - - - org.apache.hadoop - hadoop-common - - - org.apache.htrace - htrace-core - - - - - - - diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java deleted file mode 100644 index ff1e13f7959..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * The administrative API for HBase Backup. Construct an instance and call {@link #close()} - * afterwards. - *

- * BackupAdmin can be used to create backups, restore data from backups and for other - * backup-related operations. - * @since 2.0 - */ -@InterfaceAudience.Private -public interface BackupAdmin extends Closeable { - - /** - * Backup given list of tables fully. This is a synchronous operation. It returns backup id on - * success or throw exception on failure. - * @param userRequest BackupRequest instance - * @return the backup Id - */ - - String backupTables(final BackupRequest userRequest) throws IOException; - - /** - * Restore backup - * @param request restore request - * @throws IOException exception - */ - void restore(RestoreRequest request) throws IOException; - - /** - * Describe backup image command - * @param backupId backup id - * @return backup info - * @throws IOException exception - */ - BackupInfo getBackupInfo(String backupId) throws IOException; - - /** - * Delete backup image command - * @param backupIds array of backup ids - * @return total number of deleted sessions - * @throws IOException exception - */ - int deleteBackups(String[] backupIds) throws IOException; - - /** - * Merge backup images command - * @param backupIds array of backup ids of images to be merged - * The resulting backup image will have the same backup id as the most - * recent image from a list of images to be merged - * @throws IOException exception - */ - void mergeBackups(String[] backupIds) throws IOException; - - /** - * Show backup history command - * @param n last n backup sessions - * @return list of backup info objects - * @throws IOException exception - */ - List getHistory(int n) throws IOException; - - /** - * Show backup history command with filters - * @param n last n backup sessions - * @param f list of filters - * @return list of backup info objects - * @throws IOException exception - */ - List getHistory(int n, BackupInfo.Filter... f) throws IOException; - - /** - * Backup sets list command - list all backup sets. Backup set is a named group of tables. - * @return all registered backup sets - * @throws IOException exception - */ - List listBackupSets() throws IOException; - - /** - * Backup set describe command. Shows list of tables in this particular backup set. - * @param name set name - * @return backup set description or null - * @throws IOException exception - */ - BackupSet getBackupSet(String name) throws IOException; - - /** - * Delete backup set command - * @param name backup set name - * @return true, if success, false - otherwise - * @throws IOException exception - */ - boolean deleteBackupSet(String name) throws IOException; - - /** - * Add tables to backup set command - * @param name name of backup set. - * @param tables array of tables to be added to this set. - * @throws IOException exception - */ - void addToBackupSet(String name, TableName[] tables) throws IOException; - - /** - * Remove tables from backup set - * @param name name of backup set. - * @param tables array of tables to be removed from this set. - * @throws IOException exception - */ - void removeFromBackupSet(String name, TableName[] tables) throws IOException; -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupClientFactory.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupClientFactory.java deleted file mode 100644 index 68e5c110910..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupClientFactory.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.backup.impl.FullTableBackupClient; -import org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient; -import org.apache.hadoop.hbase.backup.impl.TableBackupClient; -import org.apache.hadoop.hbase.client.Connection; - -import org.apache.yetus.audience.InterfaceAudience; - -@InterfaceAudience.Private -public class BackupClientFactory { - - public static TableBackupClient create (Connection conn, String backupId, BackupRequest request) - throws IOException - { - Configuration conf = conn.getConfiguration(); - try { - String clsName = conf.get(TableBackupClient.BACKUP_CLIENT_IMPL_CLASS); - if (clsName != null) { - Class clientImpl = Class.forName(clsName); - TableBackupClient client = (TableBackupClient) clientImpl.newInstance(); - client.init(conn, backupId, request); - return client; - } - } catch (Exception e) { - throw new IOException(e); - } - - BackupType type = request.getBackupType(); - if (type == BackupType.FULL) { - return new FullTableBackupClient(conn, backupId, request); - } else { - return new IncrementalTableBackupClient(conn, backupId, request); - } - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyJob.java deleted file mode 100644 index 3385f1de774..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyJob.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Backup copy job is a part of a backup process. The concrete implementation is responsible for - * copying data from a cluster to backup destination. Concrete implementation is provided by backup - * provider, see {@link BackupRestoreFactory} - */ -@InterfaceAudience.Private -public interface BackupCopyJob extends Configurable { - - /** - * Copy backup data to destination - * @param backupInfo context object - * @param backupManager backup manager - * @param conf configuration - * @param backupType backup type (FULL or INCREMENTAL) - * @param options array of options (implementation-specific) - * @return result (0 - success, -1 failure ) - * @throws IOException exception - */ - int copy(BackupInfo backupInfo, BackupManager backupManager, Configuration conf, - BackupType backupType, String[] options) throws IOException; - - /** - * Cancel copy job - * @param jobHandler backup copy job handler - * @throws IOException - */ - void cancel(String jobHandler) throws IOException; -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java deleted file mode 100644 index 5f918f386ec..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java +++ /dev/null @@ -1,214 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; - -import java.io.IOException; -import java.net.URI; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand; -import org.apache.hadoop.hbase.backup.impl.BackupCommands; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.util.AbstractHBaseTool; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.util.ToolRunner; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -/** - * - * Command-line entry point for backup operation - * - */ -@InterfaceAudience.Private -public class BackupDriver extends AbstractHBaseTool { - - private static final Log LOG = LogFactory.getLog(BackupDriver.class); - private CommandLine cmd; - - public BackupDriver() throws IOException { - init(); - } - - protected void init() throws IOException { - // disable irrelevant loggers to avoid it mess up command output - LogUtils.disableZkAndClientLoggers(LOG); - } - - private int parseAndRun(String[] args) throws IOException { - - // Check if backup is enabled - if (!BackupManager.isBackupEnabled(getConf())) { - System.err.println(BackupRestoreConstants.ENABLE_BACKUP); - return -1; - } - - System.out.println(BackupRestoreConstants.VERIFY_BACKUP); - - String cmd = null; - String[] remainArgs = null; - if (args == null || args.length == 0) { - printToolUsage(); - return -1; - } else { - cmd = args[0]; - remainArgs = new String[args.length - 1]; - if (args.length > 1) { - System.arraycopy(args, 1, remainArgs, 0, args.length - 1); - } - } - - BackupCommand type = BackupCommand.HELP; - if (BackupCommand.CREATE.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.CREATE; - } else if (BackupCommand.HELP.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.HELP; - } else if (BackupCommand.DELETE.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.DELETE; - } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.DESCRIBE; - } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.HISTORY; - } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.PROGRESS; - } else if (BackupCommand.SET.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.SET; - } else if (BackupCommand.REPAIR.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.REPAIR; - } else if (BackupCommand.MERGE.name().equalsIgnoreCase(cmd)) { - type = BackupCommand.MERGE; - } else { - System.out.println("Unsupported command for backup: " + cmd); - printToolUsage(); - return -1; - } - - // enable debug logging - Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup"); - if (this.cmd.hasOption(OPTION_DEBUG)) { - backupClientLogger.setLevel(Level.DEBUG); - } else { - backupClientLogger.setLevel(Level.INFO); - } - - BackupCommands.Command command = BackupCommands.createCommand(getConf(), type, this.cmd); - if (type == BackupCommand.CREATE && conf != null) { - ((BackupCommands.CreateCommand) command).setConf(conf); - } - try { - command.execute(); - } catch (IOException e) { - if (e.getMessage().equals(BackupCommands.INCORRECT_USAGE)) { - return -1; - } - throw e; - } finally { - command.finish(); - } - return 0; - } - - @Override - protected void addOptions() { - // define supported options - addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC); - addOptWithArg(OPTION_TABLE, OPTION_TABLE_DESC); - addOptWithArg(OPTION_BANDWIDTH, OPTION_BANDWIDTH_DESC); - addOptWithArg(OPTION_WORKERS, OPTION_WORKERS_DESC); - addOptWithArg(OPTION_RECORD_NUMBER, OPTION_RECORD_NUMBER_DESC); - addOptWithArg(OPTION_SET, OPTION_SET_DESC); - addOptWithArg(OPTION_PATH, OPTION_PATH_DESC); - addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_DESC); - - } - - @Override - protected void processOptions(CommandLine cmd) { - this.cmd = cmd; - } - - @Override - protected int doWork() throws Exception { - return parseAndRun(cmd.getArgs()); - } - - public static void main(String[] args) throws Exception { - Configuration conf = HBaseConfiguration.create(); - Path hbasedir = FSUtils.getRootDir(conf); - URI defaultFs = hbasedir.getFileSystem(conf).getUri(); - FSUtils.setFsDefault(conf, new Path(defaultFs)); - int ret = ToolRunner.run(conf, new BackupDriver(), args); - System.exit(ret); - } - - @Override - public int run(String[] args) throws IOException { - if (conf == null) { - LOG.error("Tool configuration is not initialized"); - throw new NullPointerException("conf"); - } - - CommandLine cmd; - try { - // parse the command line arguments - cmd = parseArgs(args); - cmdLineArgs = args; - } catch (Exception e) { - System.err.println("Error when parsing command-line arguments: " + e.getMessage()); - printToolUsage(); - return EXIT_FAILURE; - } - processOptions(cmd); - - int ret = EXIT_FAILURE; - try { - ret = doWork(); - } catch (Exception e) { - LOG.error("Error running command-line tool", e); - return EXIT_FAILURE; - } - return ret; - } - - protected void printToolUsage() throws IOException { - System.out.println(BackupCommands.USAGE); - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java deleted file mode 100644 index c6aa5f00981..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hbase.shaded.com.google.common.base.Predicate; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterables; - -/** - * Implementation of a file cleaner that checks if an hfile is still referenced by backup before - * deleting it from hfile archive directory. - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class BackupHFileCleaner extends BaseHFileCleanerDelegate implements Abortable { - private static final Log LOG = LogFactory.getLog(BackupHFileCleaner.class); - private boolean stopped = false; - private boolean aborted; - private Configuration conf; - private Connection connection; - private long prevReadFromBackupTbl = 0, // timestamp of most recent read from backup:system table - secondPrevReadFromBackupTbl = 0; // timestamp of 2nd most recent read from backup:system table - //used by unit test to skip reading backup:system - private boolean checkForFullyBackedUpTables = true; - private List fullyBackedUpTables = null; - - private Set getFilenameFromBulkLoad(Map>[] maps) { - Set filenames = new HashSet(); - for (Map> map : maps) { - if (map == null) continue; - for (List paths : map.values()) { - for (Path p : paths) { - filenames.add(p.getName()); - } - } - } - return filenames; - } - - private Set loadHFileRefs(List tableList) throws IOException { - if (connection == null) { - connection = ConnectionFactory.createConnection(conf); - } - try (BackupSystemTable tbl = new BackupSystemTable(connection)) { - Map>[] res = - tbl.readBulkLoadedFiles(null, tableList); - secondPrevReadFromBackupTbl = prevReadFromBackupTbl; - prevReadFromBackupTbl = EnvironmentEdgeManager.currentTime(); - return getFilenameFromBulkLoad(res); - } - } - - @VisibleForTesting - void setCheckForFullyBackedUpTables(boolean b) { - checkForFullyBackedUpTables = b; - } - @Override - public Iterable getDeletableFiles(Iterable files) { - if (conf == null) { - return files; - } - // obtain the Set of TableName's which have been fully backed up - // so that we filter BulkLoad to be returned from server - if (checkForFullyBackedUpTables) { - if (connection == null) return files; - try (BackupSystemTable tbl = new BackupSystemTable(connection)) { - fullyBackedUpTables = tbl.getTablesForBackupType(BackupType.FULL); - } catch (IOException ioe) { - LOG.error("Failed to get tables which have been fully backed up, skipping checking", ioe); - return Collections.emptyList(); - } - Collections.sort(fullyBackedUpTables); - } - final Set hfileRefs; - try { - hfileRefs = loadHFileRefs(fullyBackedUpTables); - } catch (IOException ioe) { - LOG.error("Failed to read hfile references, skipping checking deletable files", ioe); - return Collections.emptyList(); - } - Iterable deletables = Iterables.filter(files, new Predicate() { - @Override - public boolean apply(FileStatus file) { - // If the file is recent, be conservative and wait for one more scan of backup:system table - if (file.getModificationTime() > secondPrevReadFromBackupTbl) { - return false; - } - String hfile = file.getPath().getName(); - boolean foundHFileRef = hfileRefs.contains(hfile); - return !foundHFileRef; - } - }); - return deletables; - } - - @Override - public boolean isFileDeletable(FileStatus fStat) { - // work is done in getDeletableFiles() - return true; - } - - @Override - public void setConf(Configuration config) { - this.conf = config; - this.connection = null; - try { - this.connection = ConnectionFactory.createConnection(conf); - } catch (IOException ioe) { - LOG.error("Couldn't establish connection", ioe); - } - } - - @Override - public void stop(String why) { - if (this.stopped) { - return; - } - if (this.connection != null) { - try { - this.connection.close(); - } catch (IOException ioe) { - LOG.debug("Got " + ioe + " when closing connection"); - } - } - this.stopped = true; - } - - @Override - public boolean isStopped() { - return this.stopped; - } - - @Override - public void abort(String why, Throwable e) { - LOG.warn("Aborting ReplicationHFileCleaner because " + why, e); - this.aborted = true; - stop(why); - } - - @Override - public boolean isAborted() { - return this.aborted; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java deleted file mode 100644 index 0df712e7101..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ /dev/null @@ -1,550 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos.BackupInfo.Builder; -import org.apache.hadoop.hbase.util.Bytes; - - -/** - * An object to encapsulate the information for each backup session - */ -@InterfaceAudience.Private -public class BackupInfo implements Comparable { - private static final Log LOG = LogFactory.getLog(BackupInfo.class); - - public static interface Filter { - - /** - * Filter interface - * @param info backup info - * @return true if info passes filter, false otherwise - */ - public boolean apply(BackupInfo info); - } - - /** - * Backup session states - */ - public static enum BackupState { - RUNNING, COMPLETE, FAILED, ANY; - } - - /** - * BackupPhase - phases of an ACTIVE backup session (running), when state of a backup session is - * BackupState.RUNNING - */ - public static enum BackupPhase { - REQUEST, SNAPSHOT, PREPARE_INCREMENTAL, SNAPSHOTCOPY, INCREMENTAL_COPY, STORE_MANIFEST; - } - - /** - * Backup id - */ - private String backupId; - - /** - * Backup type, full or incremental - */ - private BackupType type; - - /** - * Target root directory for storing the backup files - */ - private String backupRootDir; - - /** - * Backup state - */ - private BackupState state; - - /** - * Backup phase - */ - private BackupPhase phase = BackupPhase.REQUEST; - - /** - * Backup failure message - */ - private String failedMsg; - - /** - * Backup status map for all tables - */ - private Map backupTableInfoMap; - - /** - * Actual start timestamp of a backup process - */ - private long startTs; - - /** - * Actual end timestamp of the backup process - */ - private long completeTs; - - /** - * Total bytes of incremental logs copied - */ - private long totalBytesCopied; - - /** - * For incremental backup, a location of a backed-up hlogs - */ - private String hlogTargetDir = null; - - /** - * Incremental backup file list - */ - private List incrBackupFileList; - - /** - * New region server log timestamps for table set after distributed log roll key - table name, - * value - map of RegionServer hostname -> last log rolled timestamp - */ - private HashMap> tableSetTimestampMap; - - /** - * Backup progress in %% (0-100) - */ - private int progress; - - /** - * Number of parallel workers. -1 - system defined - */ - private int workers = -1; - - /** - * Bandwidth per worker in MB per sec. -1 - unlimited - */ - private long bandwidth = -1; - - public BackupInfo() { - backupTableInfoMap = new HashMap(); - } - - public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) { - this(); - this.backupId = backupId; - this.type = type; - this.backupRootDir = targetRootDir; - this.addTables(tables); - if (type == BackupType.INCREMENTAL) { - setHLogTargetDir(BackupUtils.getLogBackupDir(targetRootDir, backupId)); - } - this.startTs = 0; - this.completeTs = 0; - } - - public int getWorkers() { - return workers; - } - - public void setWorkers(int workers) { - this.workers = workers; - } - - public long getBandwidth() { - return bandwidth; - } - - public void setBandwidth(long bandwidth) { - this.bandwidth = bandwidth; - } - - public void setBackupTableInfoMap(Map backupTableInfoMap) { - this.backupTableInfoMap = backupTableInfoMap; - } - - public HashMap> getTableSetTimestampMap() { - return tableSetTimestampMap; - } - - public void setTableSetTimestampMap(HashMap> tableSetTimestampMap) { - this.tableSetTimestampMap = tableSetTimestampMap; - } - - public void setType(BackupType type) { - this.type = type; - } - - public void setBackupRootDir(String targetRootDir) { - this.backupRootDir = targetRootDir; - } - - public void setTotalBytesCopied(long totalBytesCopied) { - this.totalBytesCopied = totalBytesCopied; - } - - /** - * Set progress (0-100%) - * @param p progress value - */ - - public void setProgress(int p) { - this.progress = p; - } - - /** - * Get current progress - */ - public int getProgress() { - return progress; - } - - public String getBackupId() { - return backupId; - } - - public void setBackupId(String backupId) { - this.backupId = backupId; - } - - public BackupTableInfo getBackupTableInfo(TableName table) { - return this.backupTableInfoMap.get(table); - } - - public String getFailedMsg() { - return failedMsg; - } - - public void setFailedMsg(String failedMsg) { - this.failedMsg = failedMsg; - } - - public long getStartTs() { - return startTs; - } - - public void setStartTs(long startTs) { - this.startTs = startTs; - } - - public long getCompleteTs() { - return completeTs; - } - - public void setCompleteTs(long endTs) { - this.completeTs = endTs; - } - - public long getTotalBytesCopied() { - return totalBytesCopied; - } - - public BackupState getState() { - return state; - } - - public void setState(BackupState flag) { - this.state = flag; - } - - public BackupPhase getPhase() { - return phase; - } - - public void setPhase(BackupPhase phase) { - this.phase = phase; - } - - public BackupType getType() { - return type; - } - - public void setSnapshotName(TableName table, String snapshotName) { - this.backupTableInfoMap.get(table).setSnapshotName(snapshotName); - } - - public String getSnapshotName(TableName table) { - return this.backupTableInfoMap.get(table).getSnapshotName(); - } - - public List getSnapshotNames() { - List snapshotNames = new ArrayList(); - for (BackupTableInfo backupStatus : this.backupTableInfoMap.values()) { - snapshotNames.add(backupStatus.getSnapshotName()); - } - return snapshotNames; - } - - public Set getTables() { - return this.backupTableInfoMap.keySet(); - } - - public List getTableNames() { - return new ArrayList(backupTableInfoMap.keySet()); - } - - public void addTables(TableName[] tables) { - for (TableName table : tables) { - BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId); - this.backupTableInfoMap.put(table, backupStatus); - } - } - - public void setTables(List tables) { - this.backupTableInfoMap.clear(); - for (TableName table : tables) { - BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId); - this.backupTableInfoMap.put(table, backupStatus); - } - } - - public String getBackupRootDir() { - return backupRootDir; - } - - public String getTableBackupDir(TableName tableName) { - return BackupUtils.getTableBackupDir(backupRootDir, backupId, tableName); - } - - public void setHLogTargetDir(String hlogTagetDir) { - this.hlogTargetDir = hlogTagetDir; - } - - public String getHLogTargetDir() { - return hlogTargetDir; - } - - public List getIncrBackupFileList() { - return incrBackupFileList; - } - - public void setIncrBackupFileList(List incrBackupFileList) { - this.incrBackupFileList = incrBackupFileList; - } - - /** - * Set the new region server log timestamps after distributed log roll - * @param newTableSetTimestampMap table timestamp map - */ - public void - setIncrTimestampMap(HashMap> newTableSetTimestampMap) { - this.tableSetTimestampMap = newTableSetTimestampMap; - } - - /** - * Get new region server log timestamps after distributed log roll - * @return new region server log timestamps - */ - public HashMap> getIncrTimestampMap() { - return this.tableSetTimestampMap; - } - - public TableName getTableBySnapshot(String snapshotName) { - for (Entry entry : this.backupTableInfoMap.entrySet()) { - if (snapshotName.equals(entry.getValue().getSnapshotName())) { - return entry.getKey(); - } - } - return null; - } - - public BackupProtos.BackupInfo toProtosBackupInfo() { - BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder(); - builder.setBackupId(getBackupId()); - setBackupTableInfoMap(builder); - builder.setCompleteTs(getCompleteTs()); - if (getFailedMsg() != null) { - builder.setFailedMessage(getFailedMsg()); - } - if (getState() != null) { - builder.setBackupState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name())); - } - if (getPhase() != null) { - builder.setBackupPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name())); - } - - builder.setProgress(getProgress()); - builder.setStartTs(getStartTs()); - builder.setBackupRootDir(getBackupRootDir()); - builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name())); - builder.setWorkersNumber(workers); - builder.setBandwidth(bandwidth); - return builder.build(); - } - - @Override - public int hashCode() { - int hash = 33 * type.hashCode() + backupId != null ? backupId.hashCode() : 0; - if (backupRootDir != null) { - hash = 33 * hash + backupRootDir.hashCode(); - } - hash = 33 * hash + state.hashCode(); - hash = 33 * hash + phase.hashCode(); - hash = 33 * hash + (int) (startTs ^ (startTs >>> 32)); - hash = 33 * hash + (int) (completeTs ^ (completeTs >>> 32)); - hash = 33 * hash + (int) (totalBytesCopied ^ (totalBytesCopied >>> 32)); - if (hlogTargetDir != null) { - hash = 33 * hash + hlogTargetDir.hashCode(); - } - return hash; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof BackupInfo) { - BackupInfo other = (BackupInfo) obj; - try { - return Bytes.equals(toByteArray(), other.toByteArray()); - } catch (IOException e) { - LOG.error(e); - return false; - } - } else { - return false; - } - } - - @Override - public String toString() { - return backupId; - } - - public byte[] toByteArray() throws IOException { - return toProtosBackupInfo().toByteArray(); - } - - private void setBackupTableInfoMap(Builder builder) { - for (Entry entry : backupTableInfoMap.entrySet()) { - builder.addBackupTableInfo(entry.getValue().toProto()); - } - } - - public static BackupInfo fromByteArray(byte[] data) throws IOException { - return fromProto(BackupProtos.BackupInfo.parseFrom(data)); - } - - public static BackupInfo fromStream(final InputStream stream) throws IOException { - return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream)); - } - - public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { - BackupInfo context = new BackupInfo(); - context.setBackupId(proto.getBackupId()); - context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList())); - context.setCompleteTs(proto.getCompleteTs()); - if (proto.hasFailedMessage()) { - context.setFailedMsg(proto.getFailedMessage()); - } - if (proto.hasBackupState()) { - context.setState(BackupInfo.BackupState.valueOf(proto.getBackupState().name())); - } - - context.setHLogTargetDir(BackupUtils.getLogBackupDir(proto.getBackupRootDir(), - proto.getBackupId())); - - if (proto.hasBackupPhase()) { - context.setPhase(BackupPhase.valueOf(proto.getBackupPhase().name())); - } - if (proto.hasProgress()) { - context.setProgress(proto.getProgress()); - } - context.setStartTs(proto.getStartTs()); - context.setBackupRootDir(proto.getBackupRootDir()); - context.setType(BackupType.valueOf(proto.getBackupType().name())); - context.setWorkers(proto.getWorkersNumber()); - context.setBandwidth(proto.getBandwidth()); - return context; - } - - private static Map toMap(List list) { - HashMap map = new HashMap<>(); - for (BackupProtos.BackupTableInfo tbs : list) { - map.put(ProtobufUtil.toTableName(tbs.getTableName()), BackupTableInfo.convert(tbs)); - } - return map; - } - - public String getShortDescription() { - StringBuilder sb = new StringBuilder(); - sb.append("{"); - sb.append("ID=" + backupId).append(","); - sb.append("Type=" + getType()).append(","); - sb.append("Tables=" + getTableListAsString()).append(","); - sb.append("State=" + getState()).append(","); - Date date = null; - Calendar cal = Calendar.getInstance(); - cal.setTimeInMillis(getStartTs()); - date = cal.getTime(); - sb.append("Start time=" + date).append(","); - if (state == BackupState.FAILED) { - sb.append("Failed message=" + getFailedMsg()).append(","); - } else if (state == BackupState.RUNNING) { - sb.append("Phase=" + getPhase()).append(","); - } else if (state == BackupState.COMPLETE) { - cal = Calendar.getInstance(); - cal.setTimeInMillis(getCompleteTs()); - date = cal.getTime(); - sb.append("End time=" + date).append(","); - } - sb.append("Progress=" + getProgress() + "%"); - sb.append("}"); - - return sb.toString(); - } - - public String getStatusAndProgressAsString() { - StringBuilder sb = new StringBuilder(); - sb.append("id: ").append(getBackupId()).append(" state: ").append(getState()) - .append(" progress: ").append(getProgress()); - return sb.toString(); - } - - public String getTableListAsString() { - StringBuffer sb = new StringBuffer(); - sb.append("{"); - sb.append(StringUtils.join(backupTableInfoMap.keySet(), ",")); - sb.append("}"); - return sb.toString(); - } - - /** - * We use only time stamps to compare objects during sort operation - */ - @Override - public int compareTo(BackupInfo o) { - Long thisTS = - Long.valueOf(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1)); - Long otherTS = Long.valueOf(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1)); - return thisTS.compareTo(otherTS); - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupMergeJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupMergeJob.java deleted file mode 100644 index dc2e85b774e..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupMergeJob.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Backup merge operation job interface. Concrete implementation is provided by backup provider, see - * {@link BackupRestoreFactory} - */ - -@InterfaceAudience.Private -public interface BackupMergeJob extends Configurable { - - /** - * Run backup merge operation - * @param backupIds backup image ids - * @throws IOException - */ - void run(String[] backupIds) throws IOException; -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupObserver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupObserver.java deleted file mode 100644 index e2b27ff537f..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupObserver.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.RegionObserver; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * An Observer to facilitate backup operations - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class BackupObserver implements RegionCoprocessor, RegionObserver { - private static final Log LOG = LogFactory.getLog(BackupObserver.class); - - @Override - public Optional getRegionObserver() { - return Optional.of(this); - } - - @Override - public boolean postBulkLoadHFile(ObserverContext ctx, - List> stagingFamilyPaths, Map> finalPaths, - boolean hasLoaded) throws IOException { - Configuration cfg = ctx.getEnvironment().getConfiguration(); - if (!hasLoaded) { - // there is no need to record state - return hasLoaded; - } - if (finalPaths == null || !BackupManager.isBackupEnabled(cfg)) { - LOG.debug("skipping recording bulk load in postBulkLoadHFile since backup is disabled"); - return hasLoaded; - } - try (Connection connection = ConnectionFactory.createConnection(cfg); - BackupSystemTable tbl = new BackupSystemTable(connection)) { - List fullyBackedUpTables = tbl.getTablesForBackupType(BackupType.FULL); - RegionInfo info = ctx.getEnvironment().getRegionInfo(); - TableName tableName = info.getTable(); - if (!fullyBackedUpTables.contains(tableName)) { - if (LOG.isTraceEnabled()) { - LOG.trace(tableName + " has not gone thru full backup"); - } - return hasLoaded; - } - tbl.writePathsPostBulkLoad(tableName, info.getEncodedNameAsBytes(), finalPaths); - return hasLoaded; - } catch (IOException ioe) { - LOG.error("Failed to get tables which have been fully backed up", ioe); - return false; - } - } - @Override - public void preCommitStoreFile(final ObserverContext ctx, - final byte[] family, final List> pairs) throws IOException { - Configuration cfg = ctx.getEnvironment().getConfiguration(); - if (pairs == null || pairs.isEmpty() || !BackupManager.isBackupEnabled(cfg)) { - LOG.debug("skipping recording bulk load in preCommitStoreFile since backup is disabled"); - return; - } - try (Connection connection = ConnectionFactory.createConnection(cfg); - BackupSystemTable tbl = new BackupSystemTable(connection)) { - List fullyBackedUpTables = tbl.getTablesForBackupType(BackupType.FULL); - RegionInfo info = ctx.getEnvironment().getRegionInfo(); - TableName tableName = info.getTable(); - if (!fullyBackedUpTables.contains(tableName)) { - if (LOG.isTraceEnabled()) { - LOG.trace(tableName + " has not gone thru full backup"); - } - return; - } - tbl.writeFilesForBulkLoadPreCommit(tableName, info.getEncodedNameAsBytes(), family, pairs); - return; - } - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java deleted file mode 100644 index 003c0e793e2..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.util.List; - -import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * POJO class for backup request - */ -@InterfaceAudience.Private -public final class BackupRequest { - - public static class Builder { - - BackupRequest request; - - public Builder() { - request = new BackupRequest(); - } - - public Builder withBackupType(BackupType type) { - request.setBackupType(type); - return this; - } - - public Builder withTableList(List tables) { - request.setTableList(tables); - return this; - } - - public Builder withTargetRootDir(String backupDir) { - request.setTargetRootDir(backupDir); - return this; - } - - public Builder withBackupSetName(String setName) { - request.setBackupSetName(setName); - return this; - } - - public Builder withTotalTasks(int numTasks) { - request.setTotalTasks(numTasks); - return this; - } - - public Builder withBandwidthPerTasks(int bandwidth) { - request.setBandwidth(bandwidth); - return this; - } - - public Builder withYarnPoolName(String name) { - request.setYarnPoolName(name); - return this; - } - - public BackupRequest build() { - return request; - } - - } - - private BackupType type; - private List tableList; - private String targetRootDir; - private int totalTasks = -1; - private long bandwidth = -1L; - private String backupSetName; - private String yarnPoolName; - - private BackupRequest() { - } - - private BackupRequest setBackupType(BackupType type) { - this.type = type; - return this; - } - - public BackupType getBackupType() { - return this.type; - } - - private BackupRequest setTableList(List tableList) { - this.tableList = tableList; - return this; - } - - public List getTableList() { - return this.tableList; - } - - private BackupRequest setTargetRootDir(String targetRootDir) { - this.targetRootDir = targetRootDir; - return this; - } - - public String getTargetRootDir() { - return this.targetRootDir; - } - - private BackupRequest setTotalTasks(int totalTasks) { - this.totalTasks = totalTasks; - return this; - } - - public int getTotalTasks() { - return this.totalTasks; - } - - private BackupRequest setBandwidth(long bandwidth) { - this.bandwidth = bandwidth; - return this; - } - - public long getBandwidth() { - return this.bandwidth; - } - - public String getBackupSetName() { - return backupSetName; - } - - private BackupRequest setBackupSetName(String backupSetName) { - this.backupSetName = backupSetName; - return this; - } - - public String getYarnPoolName() { - return yarnPoolName; - } - - public void setYarnPoolName(String yarnPoolName) { - this.yarnPoolName = yarnPoolName; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java deleted file mode 100644 index 3ea8c56655e..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import org.apache.hadoop.hbase.HConstants; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * BackupRestoreConstants holds a bunch of HBase Backup and Restore constants - */ -@InterfaceAudience.Private -public interface BackupRestoreConstants { - /* - * Backup/Restore constants - */ - public final static String BACKUP_SYSTEM_TABLE_NAME_KEY = "hbase.backup.system.table.name"; - public final static String BACKUP_SYSTEM_TABLE_NAME_DEFAULT = "backup:system"; - - public final static String BACKUP_SYSTEM_TTL_KEY = "hbase.backup.system.ttl"; - - public final static int BACKUP_SYSTEM_TTL_DEFAULT = HConstants.FOREVER; - public final static String BACKUP_ENABLE_KEY = "hbase.backup.enable"; - public final static boolean BACKUP_ENABLE_DEFAULT = false; - - - public static final String BACKUP_MAX_ATTEMPTS_KEY = "hbase.backup.attempts.max"; - public static final int DEFAULT_BACKUP_MAX_ATTEMPTS = 10; - - public static final String BACKUP_ATTEMPTS_PAUSE_MS_KEY = "hbase.backup.attempts.pause.ms"; - public static final int DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS = 10000; - - /* - * Drivers option list - */ - public static final String OPTION_OVERWRITE = "o"; - public static final String OPTION_OVERWRITE_DESC = - "Overwrite data if any of the restore target tables exists"; - - public static final String OPTION_CHECK = "c"; - public static final String OPTION_CHECK_DESC = - "Check restore sequence and dependencies only (does not execute the command)"; - - public static final String OPTION_SET = "s"; - public static final String OPTION_SET_DESC = "Backup set name"; - public static final String OPTION_SET_RESTORE_DESC = - "Backup set to restore, mutually exclusive with -t (table list)"; - public static final String OPTION_SET_BACKUP_DESC = - "Backup set to backup, mutually exclusive with -t (table list)"; - public static final String OPTION_DEBUG = "d"; - public static final String OPTION_DEBUG_DESC = "Enable debug loggings"; - - public static final String OPTION_TABLE = "t"; - public static final String OPTION_TABLE_DESC = "Table name. If specified, only backup images," - + " which contain this table will be listed."; - - public static final String OPTION_TABLE_LIST = "l"; - public static final String OPTION_TABLE_LIST_DESC = "Table name list, comma-separated."; - - public static final String OPTION_BANDWIDTH = "b"; - public static final String OPTION_BANDWIDTH_DESC = "Bandwidth per task (MapReduce task) in MB/s"; - - public static final String OPTION_WORKERS = "w"; - public static final String OPTION_WORKERS_DESC = "Number of parallel MapReduce tasks to execute"; - - public static final String OPTION_RECORD_NUMBER = "n"; - public static final String OPTION_RECORD_NUMBER_DESC = - "Number of records of backup history. Default: 10"; - - public static final String OPTION_PATH = "p"; - public static final String OPTION_PATH_DESC = "Backup destination root directory path"; - - public static final String OPTION_TABLE_MAPPING = "m"; - public static final String OPTION_TABLE_MAPPING_DESC = - "A comma separated list of target tables. " - + "If specified, each table in must have a mapping"; - public static final String OPTION_YARN_QUEUE_NAME = "q"; - public static final String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup create command on"; - public static final String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run backup restore command on"; - - public final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; - - public static final String BACKUP_CONFIG_STRING = BackupRestoreConstants.BACKUP_ENABLE_KEY + "=true\n" - + "hbase.master.logcleaner.plugins=" - +"YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner\n" - + "hbase.procedure.master.classes=YOUR_CLASSES," - +"org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager\n" - + "hbase.procedure.regionserver.classes=YOUR_CLASSES," - + "org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager\n" - + "hbase.coprocessor.region.classes=YOUR_CLASSES," - + "org.apache.hadoop.hbase.backup.BackupObserver\n" - + "and restart the cluster\n"; - public static final String ENABLE_BACKUP = "Backup is not enabled. To enable backup, "+ - "in hbase-site.xml, set:\n " - + BACKUP_CONFIG_STRING; - - public static final String VERIFY_BACKUP = "Please make sure that backup is enabled on the cluster. To enable backup, "+ - "in hbase-site.xml, set:\n " - + BACKUP_CONFIG_STRING; - - /* - * Delimiter in table name list in restore command - */ - public static final String TABLENAME_DELIMITER_IN_COMMAND = ","; - - public static final String CONF_STAGING_ROOT = "snapshot.export.staging.root"; - - public static final String BACKUPID_PREFIX = "backup_"; - - public static enum BackupCommand { - CREATE, CANCEL, DELETE, DESCRIBE, HISTORY, STATUS, CONVERT, MERGE, STOP, SHOW, HELP, PROGRESS, - SET, SET_ADD, SET_REMOVE, SET_DELETE, SET_DESCRIBE, SET_LIST, REPAIR - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreFactory.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreFactory.java deleted file mode 100644 index 9612ebe6749..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreFactory.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyJob; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupMergeJob; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * Factory implementation for backup/restore related jobs - * - */ -@InterfaceAudience.Private -public final class BackupRestoreFactory { - - public final static String HBASE_INCR_RESTORE_IMPL_CLASS = "hbase.incremental.restore.class"; - public final static String HBASE_BACKUP_COPY_IMPL_CLASS = "hbase.backup.copy.class"; - public final static String HBASE_BACKUP_MERGE_IMPL_CLASS = "hbase.backup.merge.class"; - - private BackupRestoreFactory() { - throw new AssertionError("Instantiating utility class..."); - } - - /** - * Gets backup restore job - * @param conf configuration - * @return backup restore job instance - */ - public static RestoreJob getRestoreJob(Configuration conf) { - Class cls = - conf.getClass(HBASE_INCR_RESTORE_IMPL_CLASS, MapReduceRestoreJob.class, RestoreJob.class); - RestoreJob service = ReflectionUtils.newInstance(cls, conf); - service.setConf(conf); - return service; - } - - /** - * Gets backup copy job - * @param conf configuration - * @return backup copy job instance - */ - public static BackupCopyJob getBackupCopyJob(Configuration conf) { - Class cls = - conf.getClass(HBASE_BACKUP_COPY_IMPL_CLASS, MapReduceBackupCopyJob.class, - BackupCopyJob.class); - BackupCopyJob service = ReflectionUtils.newInstance(cls, conf); - service.setConf(conf); - return service; - } - - /** - * Gets backup merge job - * @param conf configuration - * @return backup merge job instance - */ - public static BackupMergeJob getBackupMergeJob(Configuration conf) { - Class cls = - conf.getClass(HBASE_BACKUP_MERGE_IMPL_CLASS, MapReduceBackupMergeJob.class, - BackupMergeJob.class); - BackupMergeJob service = ReflectionUtils.newInstance(cls, conf); - service.setConf(conf); - return service; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupTableInfo.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupTableInfo.java deleted file mode 100644 index 5f221a525d6..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupTableInfo.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; - -/** - * Backup related information encapsulated for a table. At this moment only target directory, - * snapshot name and table name are encapsulated here. - */ - -@InterfaceAudience.Private -public class BackupTableInfo { - - /* - * Table name for backup - */ - private TableName table; - - /* - * Snapshot name for offline/online snapshot - */ - - private String snapshotName = null; - - public BackupTableInfo() { - - } - - public BackupTableInfo(TableName table, String targetRootDir, String backupId) { - this.table = table; - } - - public String getSnapshotName() { - return snapshotName; - } - - public void setSnapshotName(String snapshotName) { - this.snapshotName = snapshotName; - } - - public TableName getTable() { - return table; - } - - public static BackupTableInfo convert(BackupProtos.BackupTableInfo proto) { - BackupTableInfo bs = new BackupTableInfo(); - bs.table = ProtobufUtil.toTableName(proto.getTableName()); - if (proto.hasSnapshotName()) { - bs.snapshotName = proto.getSnapshotName(); - } - return bs; - } - - public BackupProtos.BackupTableInfo toProto() { - BackupProtos.BackupTableInfo.Builder builder = BackupProtos.BackupTableInfo.newBuilder(); - if (snapshotName != null) { - builder.setSnapshotName(snapshotName); - } - builder.setTableName(org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toProtoTableName(table)); - return builder.build(); - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java deleted file mode 100644 index f5f879a5bd1..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java +++ /dev/null @@ -1,146 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupManifest; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * View to an on-disk Backup Image FileSytem Provides the set of methods necessary to interact with - * the on-disk Backup Image data. - */ -@InterfaceAudience.Private -public class HBackupFileSystem { - public static final Log LOG = LogFactory.getLog(HBackupFileSystem.class); - - /** - * This is utility class. - */ - private HBackupFileSystem() { - } - - /** - * Given the backup root dir, backup id and the table name, return the backup image location, - * which is also where the backup manifest file is. return value look like: - * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where - * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory - * @param backupRootDir backup root directory - * @param backupId backup id - * @param tableName table name - * @return backupPath String for the particular table - */ - public static String - getTableBackupDir(String backupRootDir, String backupId, TableName tableName) { - return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR - + tableName.getNamespaceAsString() + Path.SEPARATOR + tableName.getQualifierAsString() - + Path.SEPARATOR; - } - - public static String getTableBackupDataDir(String backupRootDir, String backupId, - TableName tableName) { - return getTableBackupDir(backupRootDir, backupId, tableName) + Path.SEPARATOR + "data"; - } - - public static Path getBackupPath(String backupRootDir, String backupId) { - return new Path(backupRootDir + Path.SEPARATOR + backupId); - } - - /** - * Given the backup root dir, backup id and the table name, return the backup image location, - * which is also where the backup manifest file is. return value look like: - * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where - * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory - * @param backupRootPath backup root path - * @param tableName table name - * @param backupId backup Id - * @return backupPath for the particular table - */ - public static Path getTableBackupPath(TableName tableName, Path backupRootPath, String backupId) { - return new Path(getTableBackupDir(backupRootPath.toString(), backupId, tableName)); - } - - /** - * Given the backup root dir and the backup id, return the log file location for an incremental - * backup. - * @param backupRootDir backup root directory - * @param backupId backup id - * @return logBackupDir: ".../user/biadmin/backup/WALs/backup_1396650096738" - */ - public static String getLogBackupDir(String backupRootDir, String backupId) { - return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR - + HConstants.HREGION_LOGDIR_NAME; - } - - public static Path getLogBackupPath(String backupRootDir, String backupId) { - return new Path(getLogBackupDir(backupRootDir, backupId)); - } - - // TODO we do not keep WAL files anymore - // Move manifest file to other place - private static Path getManifestPath(Configuration conf, Path backupRootPath, String backupId) - throws IOException { - Path manifestPath = null; - - FileSystem fs = backupRootPath.getFileSystem(conf); - manifestPath = - new Path(getBackupPath(backupRootPath.toString(), backupId) + Path.SEPARATOR - + BackupManifest.MANIFEST_FILE_NAME); - if (!fs.exists(manifestPath)) { - String errorMsg = - "Could not find backup manifest " + BackupManifest.MANIFEST_FILE_NAME + " for " - + backupId + ". File " + manifestPath + " does not exists. Did " + backupId - + " correspond to previously taken backup ?"; - throw new IOException(errorMsg); - } - return manifestPath; - } - - public static BackupManifest - getManifest(Configuration conf, Path backupRootPath, String backupId) throws IOException { - BackupManifest manifest = - new BackupManifest(conf, getManifestPath(conf, backupRootPath, backupId)); - return manifest; - } - - /** - * Check whether the backup image path and there is manifest file in the path. - * @param backupManifestMap If all the manifests are found, then they are put into this map - * @param tableArray the tables involved - * @throws IOException exception - */ - public static void checkImageManifestExist(HashMap backupManifestMap, - TableName[] tableArray, Configuration conf, Path backupRootPath, String backupId) - throws IOException { - for (TableName tableName : tableArray) { - BackupManifest manifest = getManifest(conf, backupRootPath, backupId); - backupManifestMap.put(tableName, manifest); - } - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java deleted file mode 100644 index 3f62338a920..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import org.apache.commons.logging.Log; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -/** - * Utility class for disabling Zk and client logging - * - */ -@InterfaceAudience.Private -final class LogUtils { - - private LogUtils() { - } - - /** - * Disables Zk- and HBase client logging - * @param log - */ - static void disableZkAndClientLoggers(Log log) { - // disable zookeeper log to avoid it mess up command output - Logger zkLogger = Logger.getLogger("org.apache.zookeeper"); - zkLogger.setLevel(Level.OFF); - // disable hbase zookeeper tool log to avoid it mess up command output - Logger hbaseZkLogger = Logger.getLogger("org.apache.hadoop.hbase.zookeeper"); - hbaseZkLogger.setLevel(Level.OFF); - // disable hbase client log to avoid it mess up command output - Logger hbaseClientLogger = Logger.getLogger("org.apache.hadoop.hbase.client"); - hbaseClientLogger.setLevel(Level.OFF); - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java deleted file mode 100644 index b99246b262c..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java +++ /dev/null @@ -1,275 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_CHECK; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_CHECK_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_OVERWRITE; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_OVERWRITE_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_RESTORE_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_RESTORE_DESC; - -import java.io.IOException; -import java.net.URI; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.util.AbstractHBaseTool; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.util.ToolRunner; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -/** - * - * Command-line entry point for restore operation - * - */ -@InterfaceAudience.Private -public class RestoreDriver extends AbstractHBaseTool { - - private static final Log LOG = LogFactory.getLog(RestoreDriver.class); - private CommandLine cmd; - - private static final String USAGE_STRING = - "Usage: hbase restore [options]\n" - + " backup_path Path to a backup destination root\n" - + " backup_id Backup image ID to restore\n" - + " table(s) Comma-separated list of tables to restore\n"; - - private static final String USAGE_FOOTER = ""; - - protected RestoreDriver() throws IOException { - init(); - } - - protected void init() throws IOException { - // disable irrelevant loggers to avoid it mess up command output - LogUtils.disableZkAndClientLoggers(LOG); - } - - private int parseAndRun(String[] args) throws IOException { - // Check if backup is enabled - if (!BackupManager.isBackupEnabled(getConf())) { - System.err.println(BackupRestoreConstants.ENABLE_BACKUP); - return -1; - } - - System.out.println(BackupRestoreConstants.VERIFY_BACKUP); - - // enable debug logging - Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup"); - if (cmd.hasOption(OPTION_DEBUG)) { - backupClientLogger.setLevel(Level.DEBUG); - } - - // whether to overwrite to existing table if any, false by default - boolean overwrite = cmd.hasOption(OPTION_OVERWRITE); - if (overwrite) { - LOG.debug("Found -overwrite option in restore command, " - + "will overwrite to existing table if any in the restore target"); - } - - // whether to only check the dependencies, false by default - boolean check = cmd.hasOption(OPTION_CHECK); - if (check) { - LOG.debug("Found -check option in restore command, " - + "will check and verify the dependencies"); - } - - if (cmd.hasOption(OPTION_SET) && cmd.hasOption(OPTION_TABLE)) { - System.err.println("Options -s and -t are mutaully exclusive,"+ - " you can not specify both of them."); - printToolUsage(); - return -1; - } - - if (!cmd.hasOption(OPTION_SET) && !cmd.hasOption(OPTION_TABLE)) { - System.err.println("You have to specify either set name or table list to restore"); - printToolUsage(); - return -1; - } - - if (cmd.hasOption(OPTION_YARN_QUEUE_NAME)) { - String queueName = cmd.getOptionValue(OPTION_YARN_QUEUE_NAME); - // Set system property value for MR job - System.setProperty("mapreduce.job.queuename", queueName); - } - - // parse main restore command options - String[] remainArgs = cmd.getArgs(); - if (remainArgs.length != 2) { - printToolUsage(); - return -1; - } - - String backupRootDir = remainArgs[0]; - String backupId = remainArgs[1]; - String tables = null; - String tableMapping = - cmd.hasOption(OPTION_TABLE_MAPPING) ? cmd.getOptionValue(OPTION_TABLE_MAPPING) : null; - try (final Connection conn = ConnectionFactory.createConnection(conf); - BackupAdmin client = new BackupAdminImpl(conn);) { - // Check backup set - if (cmd.hasOption(OPTION_SET)) { - String setName = cmd.getOptionValue(OPTION_SET); - try { - tables = getTablesForSet(conn, setName, conf); - } catch (IOException e) { - System.out.println("ERROR: " + e.getMessage() + " for setName=" + setName); - printToolUsage(); - return -2; - } - if (tables == null) { - System.out.println("ERROR: Backup set '" + setName - + "' is either empty or does not exist"); - printToolUsage(); - return -3; - } - } else { - tables = cmd.getOptionValue(OPTION_TABLE); - } - - TableName[] sTableArray = BackupUtils.parseTableNames(tables); - TableName[] tTableArray = BackupUtils.parseTableNames(tableMapping); - - if (sTableArray != null && tTableArray != null && - (sTableArray.length != tTableArray.length)) { - System.out.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping); - printToolUsage(); - return -4; - } - - client.restore(BackupUtils.createRestoreRequest(backupRootDir, backupId, check, - sTableArray, tTableArray, overwrite)); - } catch (Exception e) { - e.printStackTrace(); - return -5; - } - return 0; - } - - private String getTablesForSet(Connection conn, String name, Configuration conf) - throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List tables = table.describeBackupSet(name); - if (tables == null) return null; - return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); - } - } - - @Override - protected void addOptions() { - // define supported options - addOptNoArg(OPTION_OVERWRITE, OPTION_OVERWRITE_DESC); - addOptNoArg(OPTION_CHECK, OPTION_CHECK_DESC); - addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC); - addOptWithArg(OPTION_SET, OPTION_SET_RESTORE_DESC); - addOptWithArg(OPTION_TABLE, OPTION_TABLE_LIST_DESC); - addOptWithArg(OPTION_TABLE_MAPPING, OPTION_TABLE_MAPPING_DESC); - addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_RESTORE_DESC); - - } - - @Override - protected void processOptions(CommandLine cmd) { - this.cmd = cmd; - } - - @Override - protected int doWork() throws Exception { - return parseAndRun(cmd.getArgs()); - } - - public static void main(String[] args) throws Exception { - Configuration conf = HBaseConfiguration.create(); - Path hbasedir = FSUtils.getRootDir(conf); - URI defaultFs = hbasedir.getFileSystem(conf).getUri(); - FSUtils.setFsDefault(conf, new Path(defaultFs)); - int ret = ToolRunner.run(conf, new RestoreDriver(), args); - System.exit(ret); - } - - @Override - public int run(String[] args) throws IOException { - if (conf == null) { - LOG.error("Tool configuration is not initialized"); - throw new NullPointerException("conf"); - } - - CommandLine cmd; - try { - // parse the command line arguments - cmd = parseArgs(args); - cmdLineArgs = args; - } catch (Exception e) { - System.out.println("Error when parsing command-line arguments: " + e.getMessage()); - printToolUsage(); - return EXIT_FAILURE; - } - - if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION)) { - printToolUsage(); - return EXIT_FAILURE; - } - - processOptions(cmd); - - int ret = EXIT_FAILURE; - try { - ret = doWork(); - } catch (Exception e) { - LOG.error("Error running command-line tool", e); - return EXIT_FAILURE; - } - return ret; - } - - protected void printToolUsage() throws IOException { - System.out.println(USAGE_STRING); - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.setLeftPadding(2); - helpFormatter.setDescPadding(8); - helpFormatter.setWidth(100); - helpFormatter.setSyntaxPrefix("Options:"); - helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java deleted file mode 100644 index ca57e597808..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Restore operation job interface Concrete implementation is provided by backup provider, see - * {@link BackupRestoreFactory} - */ - -@InterfaceAudience.Private -public interface RestoreJob extends Configurable { - - /** - * Run restore operation - * @param dirPaths path array of WAL log directories - * @param fromTables from tables - * @param toTables to tables - * @param fullBackupRestore full backup restore - * @throws IOException - */ - void run(Path[] dirPaths, TableName[] fromTables, TableName[] toTables, - boolean fullBackupRestore) throws IOException; -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java deleted file mode 100644 index 5a0a7d4c9fc..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * POJO class for restore request - */ -@InterfaceAudience.Private -public class RestoreRequest { - - public static class Builder { - RestoreRequest request; - - public Builder() { - request = new RestoreRequest(); - } - - public Builder withBackupRootDir(String backupRootDir) { - request.setBackupRootDir(backupRootDir); - return this; - } - - public Builder withBackupId(String backupId) { - request.setBackupId(backupId); - return this; - } - - public Builder withCheck(boolean check) { - request.setCheck(check); - return this; - } - - public Builder withFromTables(TableName[] fromTables) { - request.setFromTables(fromTables); - return this; - } - - public Builder withToTables(TableName[] toTables) { - request.setToTables(toTables); - return this; - } - - public Builder withOvewrite(boolean overwrite) { - request.setOverwrite(overwrite); - return this; - } - - - public RestoreRequest build() { - return request; - } - } - - private String backupRootDir; - private String backupId; - private boolean check = false; - private TableName[] fromTables; - private TableName[] toTables; - private boolean overwrite = false; - - private RestoreRequest() { - } - - public String getBackupRootDir() { - return backupRootDir; - } - - private RestoreRequest setBackupRootDir(String backupRootDir) { - this.backupRootDir = backupRootDir; - return this; - } - - public String getBackupId() { - return backupId; - } - - private RestoreRequest setBackupId(String backupId) { - this.backupId = backupId; - return this; - } - - public boolean isCheck() { - return check; - } - - private RestoreRequest setCheck(boolean check) { - this.check = check; - return this; - } - - public TableName[] getFromTables() { - return fromTables; - } - - private RestoreRequest setFromTables(TableName[] fromTables) { - this.fromTables = fromTables; - return this; - } - - public TableName[] getToTables() { - return toTables; - } - - private RestoreRequest setToTables(TableName[] toTables) { - this.toTables = toTables; - return this; - } - - public boolean isOverwrite() { - return overwrite; - } - - private RestoreRequest setOverwrite(boolean overwrite) { - this.overwrite = overwrite; - return this; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java deleted file mode 100644 index 8a60e6717d9..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ /dev/null @@ -1,743 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupAdmin; -import org.apache.hadoop.hbase.backup.BackupClientFactory; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupMergeJob; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.RestoreRequest; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -@InterfaceAudience.Private -public class BackupAdminImpl implements BackupAdmin { - public final static String CHECK_OK = "Checking backup images: OK"; - public final static String CHECK_FAILED = - "Checking backup images: Failed. Some dependencies are missing for restore"; - private static final Log LOG = LogFactory.getLog(BackupAdminImpl.class); - - private final Connection conn; - - public BackupAdminImpl(Connection conn) { - this.conn = conn; - } - - @Override - public void close() throws IOException { - } - - @Override - public BackupInfo getBackupInfo(String backupId) throws IOException { - BackupInfo backupInfo = null; - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - if (backupId == null) { - ArrayList recentSessions = table.getBackupInfos(BackupState.RUNNING); - if (recentSessions.isEmpty()) { - LOG.warn("No ongoing sessions found."); - return null; - } - // else show status for ongoing session - // must be one maximum - return recentSessions.get(0); - } else { - backupInfo = table.readBackupInfo(backupId); - return backupInfo; - } - } - } - - @Override - public int deleteBackups(String[] backupIds) throws IOException { - - int totalDeleted = 0; - Map> allTablesMap = new HashMap>(); - - boolean deleteSessionStarted = false; - boolean snapshotDone = false; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { - - // Step 1: Make sure there is no active session - // is running by using startBackupSession API - // If there is an active session in progress, exception will be thrown - try { - sysTable.startBackupExclusiveOperation(); - deleteSessionStarted = true; - } catch (IOException e) { - LOG.warn("You can not run delete command while active backup session is in progress. \n" - + "If there is no active backup session running, run backup repair utility to restore \n" - + "backup system integrity."); - return -1; - } - - // Step 2: Make sure there is no failed session - List list = sysTable.getBackupInfos(BackupState.RUNNING); - if (list.size() != 0) { - // ailed sessions found - LOG.warn("Failed backup session found. Run backup repair tool first."); - return -1; - } - - // Step 3: Record delete session - sysTable.startDeleteOperation(backupIds); - // Step 4: Snapshot backup system table - if (!BackupSystemTable.snapshotExists(conn)) { - BackupSystemTable.snapshot(conn); - } else { - LOG.warn("Backup system table snapshot exists"); - } - snapshotDone = true; - try { - for (int i = 0; i < backupIds.length; i++) { - BackupInfo info = sysTable.readBackupInfo(backupIds[i]); - if (info != null) { - String rootDir = info.getBackupRootDir(); - HashSet allTables = allTablesMap.get(rootDir); - if (allTables == null) { - allTables = new HashSet(); - allTablesMap.put(rootDir, allTables); - } - allTables.addAll(info.getTableNames()); - totalDeleted += deleteBackup(backupIds[i], sysTable); - } - } - finalizeDelete(allTablesMap, sysTable); - // Finish - sysTable.finishDeleteOperation(); - // delete snapshot - BackupSystemTable.deleteSnapshot(conn); - } catch (IOException e) { - // Fail delete operation - // Step 1 - if (snapshotDone) { - if (BackupSystemTable.snapshotExists(conn)) { - BackupSystemTable.restoreFromSnapshot(conn); - // delete snapshot - BackupSystemTable.deleteSnapshot(conn); - // We still have record with unfinished delete operation - LOG.error("Delete operation failed, please run backup repair utility to restore " - + "backup system integrity", e); - throw e; - } else { - LOG.warn("Delete operation succeeded, there were some errors: ", e); - } - } - - } finally { - if (deleteSessionStarted) { - sysTable.finishBackupExclusiveOperation(); - } - } - } - return totalDeleted; - } - - /** - * Updates incremental backup set for every backupRoot - * @param tablesMap map [backupRoot: Set] - * @param table backup system table - * @throws IOException - */ - - private void finalizeDelete(Map> tablesMap, BackupSystemTable table) - throws IOException { - for (String backupRoot : tablesMap.keySet()) { - Set incrTableSet = table.getIncrementalBackupTableSet(backupRoot); - Map> tableMap = - table.getBackupHistoryForTableSet(incrTableSet, backupRoot); - for (Map.Entry> entry : tableMap.entrySet()) { - if (entry.getValue() == null) { - // No more backups for a table - incrTableSet.remove(entry.getKey()); - } - } - if (!incrTableSet.isEmpty()) { - table.addIncrementalBackupTableSet(incrTableSet, backupRoot); - } else { // empty - table.deleteIncrementalBackupTableSet(backupRoot); - } - } - } - - /** - * Delete single backup and all related backups
- * Algorithm:
- * Backup type: FULL or INCREMENTAL
- * Is this last backup session for table T: YES or NO
- * For every table T from table list 'tables':
- * if(FULL, YES) deletes only physical data (PD)
- * if(FULL, NO), deletes PD, scans all newer backups and removes T from backupInfo,
- * until we either reach the most recent backup for T in the system or FULL backup
- * which includes T
- * if(INCREMENTAL, YES) deletes only physical data (PD) if(INCREMENTAL, NO) deletes physical data - * and for table T scans all backup images between last
- * FULL backup, which is older than the backup being deleted and the next FULL backup (if exists)
- * or last one for a particular table T and removes T from list of backup tables. - * @param backupId backup id - * @param sysTable backup system table - * @return total number of deleted backup images - * @throws IOException - */ - private int deleteBackup(String backupId, BackupSystemTable sysTable) throws IOException { - - BackupInfo backupInfo = sysTable.readBackupInfo(backupId); - - int totalDeleted = 0; - if (backupInfo != null) { - LOG.info("Deleting backup " + backupInfo.getBackupId() + " ..."); - // Step 1: clean up data for backup session (idempotent) - BackupUtils.cleanupBackupData(backupInfo, conn.getConfiguration()); - // List of tables in this backup; - List tables = backupInfo.getTableNames(); - long startTime = backupInfo.getStartTs(); - for (TableName tn : tables) { - boolean isLastBackupSession = isLastBackupSession(sysTable, tn, startTime); - if (isLastBackupSession) { - continue; - } - // else - List affectedBackups = getAffectedBackupSessions(backupInfo, tn, sysTable); - for (BackupInfo info : affectedBackups) { - if (info.equals(backupInfo)) { - continue; - } - removeTableFromBackupImage(info, tn, sysTable); - } - } - Map map = sysTable.readBulkLoadedFiles(backupId); - FileSystem fs = FileSystem.get(conn.getConfiguration()); - boolean success = true; - int numDeleted = 0; - for (String f : map.values()) { - Path p = new Path(f); - try { - LOG.debug("Delete backup info " + p + " for " + backupInfo.getBackupId()); - if (!fs.delete(p)) { - if (fs.exists(p)) { - LOG.warn(f + " was not deleted"); - success = false; - } - } else { - numDeleted++; - } - } catch (IOException ioe) { - LOG.warn(f + " was not deleted", ioe); - success = false; - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(numDeleted + " bulk loaded files out of " + map.size() + " were deleted"); - } - if (success) { - sysTable.deleteBulkLoadedFiles(map); - } - - sysTable.deleteBackupInfo(backupInfo.getBackupId()); - LOG.info("Delete backup " + backupInfo.getBackupId() + " completed."); - totalDeleted++; - } else { - LOG.warn("Delete backup failed: no information found for backupID=" + backupId); - } - return totalDeleted; - } - - private void - removeTableFromBackupImage(BackupInfo info, TableName tn, BackupSystemTable sysTable) - throws IOException { - List tables = info.getTableNames(); - LOG.debug("Remove " + tn + " from " + info.getBackupId() + " tables=" - + info.getTableListAsString()); - if (tables.contains(tn)) { - tables.remove(tn); - - if (tables.isEmpty()) { - LOG.debug("Delete backup info " + info.getBackupId()); - - sysTable.deleteBackupInfo(info.getBackupId()); - // Idempotent operation - BackupUtils.cleanupBackupData(info, conn.getConfiguration()); - } else { - info.setTables(tables); - sysTable.updateBackupInfo(info); - // Now, clean up directory for table (idempotent) - cleanupBackupDir(info, tn, conn.getConfiguration()); - } - } - } - - private List getAffectedBackupSessions(BackupInfo backupInfo, TableName tn, - BackupSystemTable table) throws IOException { - LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); - long ts = backupInfo.getStartTs(); - List list = new ArrayList(); - List history = table.getBackupHistory(backupInfo.getBackupRootDir()); - // Scan from most recent to backupInfo - // break when backupInfo reached - for (BackupInfo info : history) { - if (info.getStartTs() == ts) { - break; - } - List tables = info.getTableNames(); - if (tables.contains(tn)) { - BackupType bt = info.getType(); - if (bt == BackupType.FULL) { - // Clear list if we encounter FULL backup - list.clear(); - } else { - LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn - + " added " + info.getBackupId() + " tables=" + info.getTableListAsString()); - list.add(info); - } - } - } - return list; - } - - /** - * Clean up the data at target directory - * @throws IOException - */ - private void cleanupBackupDir(BackupInfo backupInfo, TableName table, Configuration conf) - throws IOException { - try { - // clean up the data at target directory - String targetDir = backupInfo.getBackupRootDir(); - if (targetDir == null) { - LOG.warn("No target directory specified for " + backupInfo.getBackupId()); - return; - } - - FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); - - Path targetDirPath = - new Path(BackupUtils.getTableBackupDir(backupInfo.getBackupRootDir(), - backupInfo.getBackupId(), table)); - if (outputFs.delete(targetDirPath, true)) { - LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); - } else { - LOG.info("No data has been found in " + targetDirPath.toString() + "."); - } - - } catch (IOException e1) { - LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " for table " + table - + "at " + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + "."); - throw e1; - } - } - - private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long startTime) - throws IOException { - List history = table.getBackupHistory(); - for (BackupInfo info : history) { - List tables = info.getTableNames(); - if (!tables.contains(tn)) { - continue; - } - if (info.getStartTs() <= startTime) { - return true; - } else { - return false; - } - } - return false; - } - - @Override - public List getHistory(int n) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List history = table.getBackupHistory(); - if (history.size() <= n) return history; - List list = new ArrayList(); - for (int i = 0; i < n; i++) { - list.add(history.get(i)); - } - return list; - } - } - - @Override - public List getHistory(int n, BackupInfo.Filter... filters) throws IOException { - if (filters.length == 0) return getHistory(n); - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List history = table.getBackupHistory(); - List result = new ArrayList(); - for (BackupInfo bi : history) { - if (result.size() == n) break; - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(bi)) { - passed = false; - break; - } - } - if (passed) { - result.add(bi); - } - } - return result; - } - } - - @Override - public List listBackupSets() throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List list = table.listBackupSets(); - List bslist = new ArrayList(); - for (String s : list) { - List tables = table.describeBackupSet(s); - if (tables != null) { - bslist.add(new BackupSet(s, tables)); - } - } - return bslist; - } - } - - @Override - public BackupSet getBackupSet(String name) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List list = table.describeBackupSet(name); - if (list == null) return null; - return new BackupSet(name, list); - } - } - - @Override - public boolean deleteBackupSet(String name) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - if (table.describeBackupSet(name) == null) { - return false; - } - table.deleteBackupSet(name); - return true; - } - } - - @Override - public void addToBackupSet(String name, TableName[] tables) throws IOException { - String[] tableNames = new String[tables.length]; - try (final BackupSystemTable table = new BackupSystemTable(conn); - final Admin admin = conn.getAdmin();) { - for (int i = 0; i < tables.length; i++) { - tableNames[i] = tables[i].getNameAsString(); - if (!admin.tableExists(TableName.valueOf(tableNames[i]))) { - throw new IOException("Cannot add " + tableNames[i] + " because it doesn't exist"); - } - } - table.addToBackupSet(name, tableNames); - LOG.info("Added tables [" + StringUtils.join(tableNames, " ") + "] to '" + name - + "' backup set"); - } - } - - @Override - public void removeFromBackupSet(String name, TableName[] tables) throws IOException { - LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name + "'"); - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - table.removeFromBackupSet(name, toStringArray(tables)); - LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name - + "' completed."); - } - } - - private String[] toStringArray(TableName[] list) { - String[] arr = new String[list.length]; - for (int i = 0; i < list.length; i++) { - arr[i] = list[i].toString(); - } - return arr; - } - - @Override - public void restore(RestoreRequest request) throws IOException { - if (request.isCheck()) { - HashMap backupManifestMap = new HashMap<>(); - // check and load backup image manifest for the tables - Path rootPath = new Path(request.getBackupRootDir()); - String backupId = request.getBackupId(); - TableName[] sTableArray = request.getFromTables(); - HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, - conn.getConfiguration(), rootPath, backupId); - - // Check and validate the backup image and its dependencies - - if (BackupUtils.validate(backupManifestMap, conn.getConfiguration())) { - LOG.info(CHECK_OK); - } else { - LOG.error(CHECK_FAILED); - } - return; - } - // Execute restore request - new RestoreTablesClient(conn, request).execute(); - } - - @Override - public String backupTables(BackupRequest request) throws IOException { - BackupType type = request.getBackupType(); - String targetRootDir = request.getTargetRootDir(); - List tableList = request.getTableList(); - - String backupId = BackupRestoreConstants.BACKUPID_PREFIX + EnvironmentEdgeManager.currentTime(); - if (type == BackupType.INCREMENTAL) { - Set incrTableSet = null; - try (BackupSystemTable table = new BackupSystemTable(conn)) { - incrTableSet = table.getIncrementalBackupTableSet(targetRootDir); - } - - if (incrTableSet.isEmpty()) { - String msg = - "Incremental backup table set contains no tables. " - + "You need to run full backup first " - + (tableList != null ? "on " + StringUtils.join(tableList, ",") : ""); - - throw new IOException(msg); - } - if (tableList != null) { - tableList.removeAll(incrTableSet); - if (!tableList.isEmpty()) { - String extraTables = StringUtils.join(tableList, ","); - String msg = - "Some tables (" + extraTables + ") haven't gone through full backup. " - + "Perform full backup on " + extraTables + " first, " + "then retry the command"; - throw new IOException(msg); - } - } - tableList = Lists.newArrayList(incrTableSet); - } - if (tableList != null && !tableList.isEmpty()) { - for (TableName table : tableList) { - String targetTableBackupDir = - HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table); - Path targetTableBackupDirPath = new Path(targetTableBackupDir); - FileSystem outputFs = - FileSystem.get(targetTableBackupDirPath.toUri(), conn.getConfiguration()); - if (outputFs.exists(targetTableBackupDirPath)) { - throw new IOException("Target backup directory " + targetTableBackupDir - + " exists already."); - } - } - ArrayList nonExistingTableList = null; - try (Admin admin = conn.getAdmin();) { - for (TableName tableName : tableList) { - if (!admin.tableExists(tableName)) { - if (nonExistingTableList == null) { - nonExistingTableList = new ArrayList<>(); - } - nonExistingTableList.add(tableName); - } - } - } - if (nonExistingTableList != null) { - if (type == BackupType.INCREMENTAL) { - // Update incremental backup set - tableList = excludeNonExistingTables(tableList, nonExistingTableList); - } else { - // Throw exception only in full mode - we try to backup non-existing table - throw new IOException("Non-existing tables found in the table list: " - + nonExistingTableList); - } - } - } - - // update table list - BackupRequest.Builder builder = new BackupRequest.Builder(); - request = - builder.withBackupType(request.getBackupType()).withTableList(tableList) - .withTargetRootDir(request.getTargetRootDir()) - .withBackupSetName(request.getBackupSetName()).withTotalTasks(request.getTotalTasks()) - .withBandwidthPerTasks((int) request.getBandwidth()).build(); - - TableBackupClient client = null; - try { - client = BackupClientFactory.create(conn, backupId, request); - } catch (IOException e) { - LOG.error("There is an active session already running"); - throw e; - } - - client.execute(); - - return backupId; - } - - private List excludeNonExistingTables(List tableList, - List nonExistingTableList) { - - for (TableName table : nonExistingTableList) { - tableList.remove(table); - } - return tableList; - } - - @Override - public void mergeBackups(String[] backupIds) throws IOException { - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - checkIfValidForMerge(backupIds, sysTable); - BackupMergeJob job = BackupRestoreFactory.getBackupMergeJob(conn.getConfiguration()); - job.run(backupIds); - } - } - - /** - * Verifies that backup images are valid for merge. - * - *

    - *
  • All backups MUST be in the same destination - *
  • No FULL backups are allowed - only INCREMENTAL - *
  • All backups must be in COMPLETE state - *
  • No holes in backup list are allowed - *
- *

- * @param backupIds list of backup ids - * @param table backup system table - * @throws IOException - */ - private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) throws IOException { - String backupRoot = null; - - final Set allTables = new HashSet(); - final Set allBackups = new HashSet(); - long minTime = Long.MAX_VALUE, maxTime = Long.MIN_VALUE; - for (String backupId : backupIds) { - BackupInfo bInfo = table.readBackupInfo(backupId); - if (bInfo == null) { - String msg = "Backup session " + backupId + " not found"; - throw new IOException(msg); - } - if (backupRoot == null) { - backupRoot = bInfo.getBackupRootDir(); - } else if (!bInfo.getBackupRootDir().equals(backupRoot)) { - throw new IOException("Found different backup destinations in a list of a backup sessions \n" - + "1. " + backupRoot + "\n" + "2. " + bInfo.getBackupRootDir()); - } - if (bInfo.getType() == BackupType.FULL) { - throw new IOException("FULL backup image can not be merged for: \n" + bInfo); - } - - if (bInfo.getState() != BackupState.COMPLETE) { - throw new IOException("Backup image " + backupId - + " can not be merged becuase of its state: " + bInfo.getState()); - } - allBackups.add(backupId); - allTables.addAll(bInfo.getTableNames()); - long time = bInfo.getStartTs(); - if (time < minTime) { - minTime = time; - } - if (time > maxTime) { - maxTime = time; - } - } - - - final long startRangeTime = minTime; - final long endRangeTime = maxTime; - final String backupDest = backupRoot; - // Check we have no 'holes' in backup id list - // Filter 1 : backupRoot - // Filter 2 : time range filter - // Filter 3 : table filter - - BackupInfo.Filter destinationFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - return info.getBackupRootDir().equals(backupDest); - } - }; - - BackupInfo.Filter timeRangeFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - long time = info.getStartTs(); - return time >= startRangeTime && time <= endRangeTime ; - } - }; - - BackupInfo.Filter tableFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - List tables = info.getTableNames(); - return !Collections.disjoint(allTables, tables); - } - }; - - BackupInfo.Filter typeFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - return info.getType() == BackupType.INCREMENTAL; - } - }; - - BackupInfo.Filter stateFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - return info.getState() == BackupState.COMPLETE; - } - }; - - List allInfos = - table.getBackupHistory( -1, destinationFilter, - timeRangeFilter, tableFilter, typeFilter, stateFilter); - if (allInfos.size() != allBackups.size()) { - // Yes we have at least one hole in backup image sequence - List missingIds = new ArrayList(); - for(BackupInfo info: allInfos) { - if(allBackups.contains(info.getBackupId())) { - continue; - } - missingIds.add(info.getBackupId()); - } - String errMsg = - "Sequence of backup ids has 'holes'. The following backup images must be added:" + - org.apache.hadoop.util.StringUtils.join(",", missingIds); - throw new IOException(errMsg); - } - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java deleted file mode 100644 index 102d45d2fcb..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ /dev/null @@ -1,1019 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; - -import java.io.IOException; -import java.net.URI; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupAdmin; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * General backup commands, options and usage messages - */ - -@InterfaceAudience.Private -public final class BackupCommands { - - public final static String INCORRECT_USAGE = "Incorrect usage"; - - public final static String TOP_LEVEL_NOT_ALLOWED = - "Top level (root) folder is not allowed to be a backup destination"; - - public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n" - + "where COMMAND is one of:\n" + " create create a new backup image\n" - + " delete delete an existing backup image\n" - + " describe show the detailed information of a backup image\n" - + " history show history of all successful backups\n" - + " progress show the progress of the latest backup request\n" - + " set backup set management\n" - + " repair repair backup system table\n" - + " merge merge backup images\n" - + "Run \'hbase backup COMMAND -h\' to see help message for each command\n"; - - public static final String CREATE_CMD_USAGE = - "Usage: hbase backup create [options]\n" - + " type \"full\" to create a full backup image\n" - + " \"incremental\" to create an incremental backup image\n" - + " backup_path Full path to store the backup image\n"; - - public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress \n" - + " backup_id Backup image id (optional). If no id specified, the command will show\n" - + " progress for currently running backup session."; - public static final String NO_INFO_FOUND = "No info was found for backup id: "; - public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found."; - - public static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup describe \n" - + " backup_id Backup image id\n"; - - public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]"; - - public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete \n" - + " backup_id Backup image id\n"; - - public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n"; - - public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n" - + " name Backup set name\n" - + " tables Comma separated list of tables.\n" + "COMMAND is one of:\n" - + " add add tables to a set, create a set if needed\n" - + " remove remove tables from a set\n" - + " list list all backup sets in the system\n" - + " describe describe set\n" + " delete delete backup set\n"; - public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n" - + " backup_ids Comma separated list of backup image ids.\n"; - - public static final String USAGE_FOOTER = ""; - - public static abstract class Command extends Configured { - CommandLine cmdline; - Connection conn; - - Command(Configuration conf) { - if (conf == null) { - conf = HBaseConfiguration.create(); - } - setConf(conf); - } - - public void execute() throws IOException { - if (cmdline.hasOption("h") || cmdline.hasOption("help")) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - // Create connection - conn = ConnectionFactory.createConnection(getConf()); - if (requiresNoActiveSession()) { - // Check active session - try (BackupSystemTable table = new BackupSystemTable(conn);) { - List sessions = table.getBackupInfos(BackupState.RUNNING); - - if (sessions.size() > 0) { - System.err.println("Found backup session in a RUNNING state: "); - System.err.println(sessions.get(0)); - System.err.println("This may indicate that a previous session has failed abnormally."); - System.err.println("In this case, backup recovery is recommended."); - throw new IOException("Active session found, aborted command execution"); - } - } - } - if (requiresConsistentState()) { - // Check failed delete - try (BackupSystemTable table = new BackupSystemTable(conn);) { - String[] ids = table.getListOfBackupIdsFromDeleteOperation(); - - if (ids != null && ids.length > 0) { - System.err.println("Found failed backup DELETE coommand. "); - System.err.println("Backup system recovery is required."); - throw new IOException("Failed backup DELETE found, aborted command execution"); - } - - ids = table.getListOfBackupIdsFromMergeOperation(); - if (ids != null && ids.length > 0) { - System.err.println("Found failed backup MERGE coommand. "); - System.err.println("Backup system recovery is required."); - throw new IOException("Failed backup MERGE found, aborted command execution"); - } - - } - } - } - - public void finish() throws IOException { - if (conn != null) { - conn.close(); - } - } - - protected abstract void printUsage(); - - /** - * The command can't be run if active backup session is in progress - * @return true if no active sessions are in progress - */ - protected boolean requiresNoActiveSession() { - return false; - } - - /** - * Command requires consistent state of a backup system Backup system may become inconsistent - * because of an abnormal termination of a backup session or delete command - * @return true, if yes - */ - protected boolean requiresConsistentState() { - return false; - } - } - - private BackupCommands() { - throw new AssertionError("Instantiating utility class..."); - } - - public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { - Command cmd = null; - switch (type) { - case CREATE: - cmd = new CreateCommand(conf, cmdline); - break; - case DESCRIBE: - cmd = new DescribeCommand(conf, cmdline); - break; - case PROGRESS: - cmd = new ProgressCommand(conf, cmdline); - break; - case DELETE: - cmd = new DeleteCommand(conf, cmdline); - break; - case HISTORY: - cmd = new HistoryCommand(conf, cmdline); - break; - case SET: - cmd = new BackupSetCommand(conf, cmdline); - break; - case REPAIR: - cmd = new RepairCommand(conf, cmdline); - break; - case MERGE: - cmd = new MergeCommand(conf, cmdline); - break; - case HELP: - default: - cmd = new HelpCommand(conf, cmdline); - break; - } - return cmd; - } - - static int numOfArgs(String[] args) { - if (args == null) return 0; - return args.length; - } - - public static class CreateCommand extends Command { - - CreateCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - protected boolean requiresNoActiveSession() { - return true; - } - - @Override - protected boolean requiresConsistentState() { - return true; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String[] args = cmdline.getArgs(); - if (args.length != 3) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - if (!BackupType.FULL.toString().equalsIgnoreCase(args[1]) - && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) { - System.out.println("ERROR: invalid backup type: " + args[1]); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - if (!verifyPath(args[2])) { - System.out.println("ERROR: invalid backup destination: " + args[2]); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String targetBackupDir = args[2]; - // Check if backup destination is top level (root) folder - not allowed - if (isRootFolder(targetBackupDir)) { - throw new IOException(TOP_LEVEL_NOT_ALLOWED); - } - String tables = null; - - // Check if we have both: backup set and list of tables - if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) { - System.out.println("ERROR: You can specify either backup set or list" - + " of tables, but not both"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - // Creates connection - super.execute(); - // Check backup set - String setName = null; - if (cmdline.hasOption(OPTION_SET)) { - setName = cmdline.getOptionValue(OPTION_SET); - tables = getTablesForSet(setName, getConf()); - - if (tables == null) { - System.out.println("ERROR: Backup set '" + setName - + "' is either empty or does not exist"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } else { - tables = cmdline.getOptionValue(OPTION_TABLE); - } - int bandwidth = - cmdline.hasOption(OPTION_BANDWIDTH) ? Integer.parseInt(cmdline - .getOptionValue(OPTION_BANDWIDTH)) : -1; - int workers = - cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline - .getOptionValue(OPTION_WORKERS)) : -1; - - if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) { - String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME); - // Set system property value for MR job - System.setProperty("mapreduce.job.queuename", queueName); - } - - try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { - - BackupRequest.Builder builder = new BackupRequest.Builder(); - BackupRequest request = - builder - .withBackupType(BackupType.valueOf(args[1].toUpperCase())) - .withTableList( - tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null) - .withTargetRootDir(targetBackupDir).withTotalTasks(workers) - .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build(); - String backupId = admin.backupTables(request); - System.out.println("Backup session " + backupId + " finished. Status: SUCCESS"); - } catch (IOException e) { - System.out.println("Backup session finished. Status: FAILURE"); - throw e; - } - } - - private boolean isRootFolder(String targetBackupDir) { - Path p = new Path(targetBackupDir); - return p.isRoot(); - } - - private boolean verifyPath(String path) { - try { - Path p = new Path(path); - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - URI uri = p.toUri(); - if (uri.getScheme() == null) return false; - FileSystem.get(uri, conf); - return true; - } catch (Exception e) { - return false; - } - } - - private String getTablesForSet(String name, Configuration conf) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List tables = table.describeBackupSet(name); - if (tables == null) return null; - return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); - } - } - - @Override - protected void printUsage() { - System.out.println(CREATE_CMD_USAGE); - Options options = new Options(); - options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC); - options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC); - options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC); - options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC); - options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC); - options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC); - - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.setLeftPadding(2); - helpFormatter.setDescPadding(8); - helpFormatter.setWidth(100); - helpFormatter.setSyntaxPrefix("Options:"); - helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); - - } - } - - private static class HelpCommand extends Command { - - HelpCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] args = cmdline.getArgs(); - if (args == null || args.length == 0) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - if (args.length != 2) { - System.out.println("ERROR: Only supports help message of a single command type"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String type = args[1]; - - if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { - System.out.println(CREATE_CMD_USAGE); - } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { - System.out.println(DESCRIBE_CMD_USAGE); - } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { - System.out.println(HISTORY_CMD_USAGE); - } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { - System.out.println(PROGRESS_CMD_USAGE); - } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { - System.out.println(DELETE_CMD_USAGE); - } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) { - System.out.println(SET_CMD_USAGE); - } else { - System.out.println("Unknown command : " + type); - printUsage(); - } - } - - @Override - protected void printUsage() { - System.out.println(USAGE); - } - } - - private static class DescribeCommand extends Command { - - DescribeCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String[] args = cmdline.getArgs(); - if (args.length != 2) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - super.execute(); - - String backupId = args[1]; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - BackupInfo info = sysTable.readBackupInfo(backupId); - if (info == null) { - System.out.println("ERROR: " + backupId + " does not exist"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - System.out.println(info.getShortDescription()); - } - } - - @Override - protected void printUsage() { - System.out.println(DESCRIBE_CMD_USAGE); - } - } - - private static class ProgressCommand extends Command { - - ProgressCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) { - System.out.println("No backup id was specified, " - + "will retrieve the most recent (ongoing) session"); - } - String[] args = cmdline == null ? null : cmdline.getArgs(); - if (args != null && args.length > 2) { - System.err.println("ERROR: wrong number of arguments: " + args.length); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - super.execute(); - - String backupId = (args == null || args.length <= 1) ? null : args[1]; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - BackupInfo info = null; - - if (backupId != null) { - info = sysTable.readBackupInfo(backupId); - } else { - List infos = sysTable.getBackupInfos(BackupState.RUNNING); - if (infos != null && infos.size() > 0) { - info = infos.get(0); - backupId = info.getBackupId(); - System.out.println("Found ongoing session with backupId=" + backupId); - } else { - } - } - int progress = info == null ? -1 : info.getProgress(); - if (progress < 0) { - if (backupId != null) { - System.out.println(NO_INFO_FOUND + backupId); - } else { - System.err.println(NO_ACTIVE_SESSION_FOUND); - } - } else { - System.out.println(backupId + " progress=" + progress + "%"); - } - } - } - - @Override - protected void printUsage() { - System.out.println(PROGRESS_CMD_USAGE); - } - } - - private static class DeleteCommand extends Command { - - DeleteCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - protected boolean requiresNoActiveSession() { - return true; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - super.execute(); - - String[] args = cmdline.getArgs(); - String[] backupIds = new String[args.length - 1]; - System.arraycopy(args, 1, backupIds, 0, backupIds.length); - try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { - int deleted = admin.deleteBackups(backupIds); - System.out.println("Deleted " + deleted + " backups. Total requested: " + (args.length -1)); - } catch (IOException e) { - System.err - .println("Delete command FAILED. Please run backup repair tool to restore backup system integrity"); - throw e; - } - - } - - @Override - protected void printUsage() { - System.out.println(DELETE_CMD_USAGE); - } - } - - private static class RepairCommand extends Command { - - RepairCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - - String[] args = cmdline == null ? null : cmdline.getArgs(); - if (args != null && args.length > 1) { - System.err.println("ERROR: wrong number of arguments: " + args.length); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - - // Failed backup - BackupInfo backupInfo; - List list = sysTable.getBackupInfos(BackupState.RUNNING); - if (list.size() == 0) { - // No failed sessions found - System.out.println("REPAIR status: no failed sessions found." - + " Checking failed delete backup operation ..."); - repairFailedBackupDeletionIfAny(conn, sysTable); - repairFailedBackupMergeIfAny(conn, sysTable); - return; - } - backupInfo = list.get(0); - // If this is a cancel exception, then we've already cleaned. - // set the failure timestamp of the overall backup - backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); - // set failure message - backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo); - // set overall backup status: failed - backupInfo.setState(BackupState.FAILED); - // compose the backup failed data - String backupFailedData = - "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs() - + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" - + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg(); - System.out.println(backupFailedData); - TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf); - // If backup session is updated to FAILED state - means we - // processed recovery already. - sysTable.updateBackupInfo(backupInfo); - sysTable.finishBackupExclusiveOperation(); - System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo); - - } - } - - private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable) - throws IOException { - String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation(); - if (backupIds == null || backupIds.length == 0) { - System.out.println("No failed backup DELETE operation found"); - // Delete backup table snapshot if exists - BackupSystemTable.deleteSnapshot(conn); - return; - } - System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds)); - System.out.println("Running DELETE again ..."); - // Restore table from snapshot - BackupSystemTable.restoreFromSnapshot(conn); - // Finish previous failed session - sysTable.finishBackupExclusiveOperation(); - try (BackupAdmin admin = new BackupAdminImpl(conn);) { - admin.deleteBackups(backupIds); - } - System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds)); - - } - - private void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable) - throws IOException { - String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation(); - if (backupIds == null || backupIds.length == 0) { - System.out.println("No failed backup MERGE operation found"); - // Delete backup table snapshot if exists - BackupSystemTable.deleteSnapshot(conn); - return; - } - System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds)); - System.out.println("Running MERGE again ..."); - // Restore table from snapshot - BackupSystemTable.restoreFromSnapshot(conn); - // Unlock backupo system - sysTable.finishBackupExclusiveOperation(); - // Finish previous failed session - sysTable.finishMergeOperation(); - try (BackupAdmin admin = new BackupAdminImpl(conn);) { - admin.mergeBackups(backupIds); - } - System.out.println("MERGE operation finished OK: " + StringUtils.join(backupIds)); - - } - - @Override - protected void printUsage() { - System.out.println(REPAIR_CMD_USAGE); - } - } - - private static class MergeCommand extends Command { - - MergeCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - protected boolean requiresNoActiveSession() { - return true; - } - - @Override - protected boolean requiresConsistentState() { - return true; - } - - @Override - public void execute() throws IOException { - super.execute(); - - String[] args = cmdline == null ? null : cmdline.getArgs(); - if (args == null || (args.length != 2)) { - System.err.println("ERROR: wrong number of arguments: " - + (args == null ? null : args.length)); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] backupIds = args[1].split(","); - if (backupIds.length < 2) { - String msg = "ERROR: can not merge a single backup image. "+ - "Number of images must be greater than 1."; - System.err.println(msg); - throw new IOException(msg); - - } - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - admin.mergeBackups(backupIds); - } - } - - @Override - protected void printUsage() { - System.out.println(MERGE_CMD_USAGE); - } - } - - private static class HistoryCommand extends Command { - - private final static int DEFAULT_HISTORY_LENGTH = 10; - - HistoryCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - - int n = parseHistoryLength(); - final TableName tableName = getTableName(); - final String setName = getTableSetName(); - BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (tableName == null) return true; - List names = info.getTableNames(); - return names.contains(tableName); - } - }; - BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (setName == null) return true; - String backupId = info.getBackupId(); - return backupId.startsWith(setName); - } - }; - Path backupRootPath = getBackupRootPath(); - List history = null; - if (backupRootPath == null) { - // Load from backup system table - super.execute(); - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter); - } - } else { - // load from backup FS - history = - BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter); - } - for (BackupInfo info : history) { - System.out.println(info.getShortDescription()); - } - } - - private Path getBackupRootPath() throws IOException { - String value = null; - try { - value = cmdline.getOptionValue(OPTION_PATH); - if (value == null) return null; - return new Path(value); - } catch (IllegalArgumentException e) { - System.out.println("ERROR: Illegal argument for backup root path: " + value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - private TableName getTableName() throws IOException { - String value = cmdline.getOptionValue(OPTION_TABLE); - if (value == null) return null; - try { - return TableName.valueOf(value); - } catch (IllegalArgumentException e) { - System.out.println("Illegal argument for table name: " + value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - private String getTableSetName() throws IOException { - String value = cmdline.getOptionValue(OPTION_SET); - return value; - } - - private int parseHistoryLength() throws IOException { - String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER); - try { - if (value == null) return DEFAULT_HISTORY_LENGTH; - return Integer.parseInt(value); - } catch (NumberFormatException e) { - System.out.println("Illegal argument for history length: " + value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - @Override - protected void printUsage() { - System.out.println(HISTORY_CMD_USAGE); - Options options = new Options(); - options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC); - options.addOption(OPTION_PATH, true, OPTION_PATH_DESC); - options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC); - options.addOption(OPTION_SET, true, OPTION_SET_DESC); - - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.setLeftPadding(2); - helpFormatter.setDescPadding(8); - helpFormatter.setWidth(100); - helpFormatter.setSyntaxPrefix("Options:"); - helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); - } - } - - private static class BackupSetCommand extends Command { - private final static String SET_ADD_CMD = "add"; - private final static String SET_REMOVE_CMD = "remove"; - private final static String SET_DELETE_CMD = "delete"; - private final static String SET_DESCRIBE_CMD = "describe"; - private final static String SET_LIST_CMD = "list"; - - BackupSetCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - // Command-line must have at least one element - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] args = cmdline.getArgs(); - String cmdStr = args[1]; - BackupCommand cmd = getCommand(cmdStr); - - switch (cmd) { - case SET_ADD: - processSetAdd(args); - break; - case SET_REMOVE: - processSetRemove(args); - break; - case SET_DELETE: - processSetDelete(args); - break; - case SET_DESCRIBE: - processSetDescribe(args); - break; - case SET_LIST: - processSetList(args); - break; - default: - break; - - } - } - - private void processSetList(String[] args) throws IOException { - super.execute(); - - // List all backup set names - // does not expect any args - try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { - List list = admin.listBackupSets(); - for (BackupSet bs : list) { - System.out.println(bs); - } - } - } - - private void processSetDescribe(String[] args) throws IOException { - if (args == null || args.length != 3) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - List tables = sysTable.describeBackupSet(setName); - BackupSet set = tables == null ? null : new BackupSet(setName, tables); - if (set == null) { - System.out.println("Set '" + setName + "' does not exist."); - } else { - System.out.println(set); - } - } - } - - private void processSetDelete(String[] args) throws IOException { - if (args == null || args.length != 3) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - boolean result = admin.deleteBackupSet(setName); - if (result) { - System.out.println("Delete set " + setName + " OK."); - } else { - System.out.println("Set " + setName + " does not exist"); - } - } - } - - private void processSetRemove(String[] args) throws IOException { - if (args == null || args.length != 4) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - String[] tables = args[3].split(","); - TableName[] tableNames = toTableNames(tables); - try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - admin.removeFromBackupSet(setName, tableNames); - } - } - - private TableName[] toTableNames(String[] tables) { - TableName[] arr = new TableName[tables.length]; - for (int i = 0; i < tables.length; i++) { - arr[i] = TableName.valueOf(tables[i]); - } - return arr; - } - - private void processSetAdd(String[] args) throws IOException { - if (args == null || args.length != 4) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - String[] tables = args[3].split(","); - TableName[] tableNames = new TableName[tables.length]; - for (int i = 0; i < tables.length; i++) { - tableNames[i] = TableName.valueOf(tables[i]); - } - try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - admin.addToBackupSet(setName, tableNames); - } - - } - - private BackupCommand getCommand(String cmdStr) throws IOException { - if (cmdStr.equals(SET_ADD_CMD)) { - return BackupCommand.SET_ADD; - } else if (cmdStr.equals(SET_REMOVE_CMD)) { - return BackupCommand.SET_REMOVE; - } else if (cmdStr.equals(SET_DELETE_CMD)) { - return BackupCommand.SET_DELETE; - } else if (cmdStr.equals(SET_DESCRIBE_CMD)) { - return BackupCommand.SET_DESCRIBE; - } else if (cmdStr.equals(SET_LIST_CMD)) { - return BackupCommand.SET_LIST; - } else { - System.out.println("ERROR: Unknown command for 'set' :" + cmdStr); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - @Override - protected void printUsage() { - System.out.println(SET_CMD_USAGE); - } - - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java deleted file mode 100644 index 0147c292a27..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Backup exception - */ -@SuppressWarnings("serial") -@InterfaceAudience.Private -public class BackupException extends HBaseIOException { - private BackupInfo info; - - /** - * Some exception happened for a backup and don't even know the backup that it was about - * @param msg Full description of the failure - */ - public BackupException(String msg) { - super(msg); - } - - /** - * Some exception happened for a backup with a cause - * @param cause the cause - */ - public BackupException(Throwable cause) { - super(cause); - } - - /** - * Exception for the given backup that has no previous root cause - * @param msg reason why the backup failed - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, BackupInfo desc) { - super(msg); - this.info = desc; - } - - /** - * Exception for the given backup due to another exception - * @param msg reason why the backup failed - * @param cause root cause of the failure - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, Throwable cause, BackupInfo desc) { - super(msg, cause); - this.info = desc; - } - - /** - * Exception when the description of the backup cannot be determined, due to some other root - * cause - * @param message description of what caused the failure - * @param e root cause - */ - public BackupException(String message, Exception e) { - super(message, e); - } - - public BackupInfo getBackupInfo() { - return this.info; - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java deleted file mode 100644 index 7ac94d8e4a8..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ /dev/null @@ -1,502 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; -import org.apache.hadoop.hbase.backup.master.BackupLogCleaner; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.procedure.ProcedureManagerHost; -import org.apache.hadoop.hbase.util.Pair; - -import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; - -/** - * Handles backup requests, creates backup info records in backup system table to - * keep track of backup sessions, dispatches backup request. - */ -@InterfaceAudience.Private -public class BackupManager implements Closeable { - private static final Log LOG = LogFactory.getLog(BackupManager.class); - - protected Configuration conf = null; - protected BackupInfo backupInfo = null; - protected BackupSystemTable systemTable; - protected final Connection conn; - - /** - * Backup manager constructor. - * @param conn connection - * @param conf configuration - * @throws IOException exception - */ - public BackupManager(Connection conn, Configuration conf) throws IOException { - if (!conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, - BackupRestoreConstants.BACKUP_ENABLE_DEFAULT)) { - throw new BackupException("HBase backup is not enabled. Check your " - + BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting."); - } - this.conf = conf; - this.conn = conn; - this.systemTable = new BackupSystemTable(conn); - - } - - /** - * Returns backup info - */ - protected BackupInfo getBackupInfo() { - return backupInfo; - } - - /** - * This method modifies the master's configuration in order to inject backup-related features - * (TESTs only) - * @param conf configuration - */ - @VisibleForTesting - public static void decorateMasterConfiguration(Configuration conf) { - if (!isBackupEnabled(conf)) { - return; - } - // Add WAL archive cleaner plug-in - String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS); - String cleanerClass = BackupLogCleaner.class.getCanonicalName(); - if (!plugins.contains(cleanerClass)) { - conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass); - } - - String classes = conf.get(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY); - String masterProcedureClass = LogRollMasterProcedureManager.class.getName(); - if (classes == null) { - conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, masterProcedureClass); - } else if (!classes.contains(masterProcedureClass)) { - conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, classes + "," + masterProcedureClass); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Added log cleaner: " + cleanerClass + "\n" + "Added master procedure manager: " - + masterProcedureClass); - } - - } - - /** - * This method modifies the Region Server configuration in order to inject backup-related features - * TESTs only. - * @param conf configuration - */ - @VisibleForTesting - public static void decorateRegionServerConfiguration(Configuration conf) { - if (!isBackupEnabled(conf)) { - return; - } - - String classes = conf.get(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY); - String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName(); - if (classes == null) { - conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, regionProcedureClass); - } else if (!classes.contains(regionProcedureClass)) { - conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, classes + "," - + regionProcedureClass); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Added region procedure manager: " + regionProcedureClass); - } - - } - - public static boolean isBackupEnabled(Configuration conf) { - return conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, - BackupRestoreConstants.BACKUP_ENABLE_DEFAULT); - } - - /** - * Get configuration - * @return configuration - */ - Configuration getConf() { - return conf; - } - - /** - * Stop all the work of backup. - */ - @Override - public void close() { - - if (systemTable != null) { - try { - systemTable.close(); - } catch (Exception e) { - LOG.error(e); - } - } - } - - /** - * Creates a backup info based on input backup request. - * @param backupId backup id - * @param type type - * @param tableList table list - * @param targetRootDir root dir - * @param workers number of parallel workers - * @param bandwidth bandwidth per worker in MB per sec - * @return BackupInfo - * @throws BackupException exception - */ - public BackupInfo createBackupInfo(String backupId, BackupType type, List tableList, - String targetRootDir, int workers, long bandwidth) throws BackupException { - if (targetRootDir == null) { - throw new BackupException("Wrong backup request parameter: target backup root directory"); - } - - if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) { - // If table list is null for full backup, which means backup all tables. Then fill the table - // list with all user tables from meta. It no table available, throw the request exception. - - HTableDescriptor[] htds = null; - try (Admin admin = conn.getAdmin()) { - htds = admin.listTables(); - } catch (Exception e) { - throw new BackupException(e); - } - - if (htds == null) { - throw new BackupException("No table exists for full backup of all tables."); - } else { - tableList = new ArrayList<>(); - for (HTableDescriptor hTableDescriptor : htds) { - TableName tn = hTableDescriptor.getTableName(); - if (tn.equals(BackupSystemTable.getTableName(conf))) { - // skip backup system table - continue; - } - tableList.add(hTableDescriptor.getTableName()); - } - - LOG.info("Full backup all the tables available in the cluster: " + tableList); - } - } - - // there are one or more tables in the table list - backupInfo = - new BackupInfo(backupId, type, tableList.toArray(new TableName[tableList.size()]), - targetRootDir); - backupInfo.setBandwidth(bandwidth); - backupInfo.setWorkers(workers); - return backupInfo; - } - - /** - * Check if any ongoing backup. Currently, we only reply on checking status in backup system - * table. We need to consider to handle the case of orphan records in the future. Otherwise, all - * the coming request will fail. - * @return the ongoing backup id if on going backup exists, otherwise null - * @throws IOException exception - */ - private String getOngoingBackupId() throws IOException { - - ArrayList sessions = systemTable.getBackupInfos(BackupState.RUNNING); - if (sessions.size() == 0) { - return null; - } - return sessions.get(0).getBackupId(); - } - - /** - * Start the backup manager service. - * @throws IOException exception - */ - public void initialize() throws IOException { - String ongoingBackupId = this.getOngoingBackupId(); - if (ongoingBackupId != null) { - LOG.info("There is a ongoing backup " + ongoingBackupId - + ". Can not launch new backup until no ongoing backup remains."); - throw new BackupException("There is ongoing backup."); - } - } - - public void setBackupInfo(BackupInfo backupInfo) { - this.backupInfo = backupInfo; - } - - /** - * Get direct ancestors of the current backup. - * @param backupInfo The backup info for the current backup - * @return The ancestors for the current backup - * @throws IOException exception - * @throws BackupException exception - */ - public ArrayList getAncestors(BackupInfo backupInfo) throws IOException, - BackupException { - LOG.debug("Getting the direct ancestors of the current backup " + backupInfo.getBackupId()); - - ArrayList ancestors = new ArrayList(); - - // full backup does not have ancestor - if (backupInfo.getType() == BackupType.FULL) { - LOG.debug("Current backup is a full backup, no direct ancestor for it."); - return ancestors; - } - - // get all backup history list in descending order - - ArrayList allHistoryList = getBackupHistory(true); - for (BackupInfo backup : allHistoryList) { - - BackupImage.Builder builder = BackupImage.newBuilder(); - - BackupImage image = - builder.withBackupId(backup.getBackupId()).withType(backup.getType()) - .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames()) - .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); - - // add the full backup image as an ancestor until the last incremental backup - if (backup.getType().equals(BackupType.FULL)) { - // check the backup image coverage, if previous image could be covered by the newer ones, - // then no need to add - if (!BackupManifest.canCoverImage(ancestors, image)) { - ancestors.add(image); - } - } else { - // found last incremental backup, if previously added full backup ancestor images can cover - // it, then this incremental ancestor is not the dependent of the current incremental - // backup, that is to say, this is the backup scope boundary of current table set. - // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing - // incremental backup - if (BackupManifest.canCoverImage(ancestors, image)) { - LOG.debug("Met the backup boundary of the current table set:"); - for (BackupImage image1 : ancestors) { - LOG.debug(" BackupID=" + image1.getBackupId() + ", BackupDir=" + image1.getRootDir()); - } - } else { - Path logBackupPath = - HBackupFileSystem.getBackupPath(backup.getBackupRootDir(), backup.getBackupId()); - LOG.debug("Current backup has an incremental backup ancestor, " - + "touching its image manifest in " + logBackupPath.toString() - + " to construct the dependency."); - BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath); - BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage(); - ancestors.add(lastIncrImage); - - LOG.debug("Last dependent incremental backup image: " + "{BackupID=" - + lastIncrImage.getBackupId() + "," + "BackupDir=" + lastIncrImage.getRootDir() + "}"); - } - } - } - LOG.debug("Got " + ancestors.size() + " ancestors for the current backup."); - return ancestors; - } - - /** - * Get the direct ancestors of this backup for one table involved. - * @param backupInfo backup info - * @param table table - * @return backupImages on the dependency list - * @throws BackupException exception - * @throws IOException exception - */ - public ArrayList getAncestors(BackupInfo backupInfo, TableName table) - throws BackupException, IOException { - ArrayList ancestors = getAncestors(backupInfo); - ArrayList tableAncestors = new ArrayList(); - for (BackupImage image : ancestors) { - if (image.hasTable(table)) { - tableAncestors.add(image); - if (image.getType() == BackupType.FULL) { - break; - } - } - } - return tableAncestors; - } - - /* - * backup system table operations - */ - - /** - * Updates status (state) of a backup session in a persistent store - * @param context context - * @throws IOException exception - */ - public void updateBackupInfo(BackupInfo context) throws IOException { - systemTable.updateBackupInfo(context); - } - - /** - * Starts new backup session - * @throws IOException if active session already exists - */ - public void startBackupSession() throws IOException { - systemTable.startBackupExclusiveOperation(); - } - - /** - * Finishes active backup session - * @throws IOException if no active session - */ - public void finishBackupSession() throws IOException { - systemTable.finishBackupExclusiveOperation(); - } - - /** - * Read the last backup start code (timestamp) of last successful backup. Will return null if - * there is no startcode stored in backup system table or the value is of length 0. These two - * cases indicate there is no successful backup completed so far. - * @return the timestamp of a last successful backup - * @throws IOException exception - */ - public String readBackupStartCode() throws IOException { - return systemTable.readBackupStartCode(backupInfo.getBackupRootDir()); - } - - /** - * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte. - * @param startCode start code - * @throws IOException exception - */ - public void writeBackupStartCode(Long startCode) throws IOException { - systemTable.writeBackupStartCode(startCode, backupInfo.getBackupRootDir()); - } - - /** - * Get the RS log information after the last log roll from backup system table. - * @return RS log info - * @throws IOException exception - */ - public HashMap readRegionServerLastLogRollResult() throws IOException { - return systemTable.readRegionServerLastLogRollResult(backupInfo.getBackupRootDir()); - } - - public Pair>>>>, List> - readBulkloadRows(List tableList) throws IOException { - return systemTable.readBulkloadRows(tableList); - } - - public void removeBulkLoadedRows(List lst, List rows) throws IOException { - systemTable.removeBulkLoadedRows(lst, rows); - } - - public void writeBulkLoadedFiles(List sTableList, Map>[] maps) - throws IOException { - systemTable.writeBulkLoadedFiles(sTableList, maps, backupInfo.getBackupId()); - } - - /** - * Get all completed backup information (in desc order by time) - * @return history info of BackupCompleteData - * @throws IOException exception - */ - public List getBackupHistory() throws IOException { - return systemTable.getBackupHistory(); - } - - public ArrayList getBackupHistory(boolean completed) throws IOException { - return systemTable.getBackupHistory(completed); - } - - /** - * Write the current timestamps for each regionserver to backup system table after a successful - * full or incremental backup. Each table may have a different set of log timestamps. The saved - * timestamp is of the last log file that was backed up already. - * @param tables tables - * @throws IOException exception - */ - public void writeRegionServerLogTimestamp(Set tables, - HashMap newTimestamps) throws IOException { - systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, backupInfo.getBackupRootDir()); - } - - /** - * Read the timestamp for each region server log after the last successful backup. Each table has - * its own set of the timestamps. - * @return the timestamp for each region server. key: tableName value: - * RegionServer,PreviousTimeStamp - * @throws IOException exception - */ - public HashMap> readLogTimestampMap() throws IOException { - return systemTable.readLogTimestampMap(backupInfo.getBackupRootDir()); - } - - /** - * Return the current tables covered by incremental backup. - * @return set of tableNames - * @throws IOException exception - */ - public Set getIncrementalBackupTableSet() throws IOException { - return systemTable.getIncrementalBackupTableSet(backupInfo.getBackupRootDir()); - } - - /** - * Adds set of tables to overall incremental backup table set - * @param tables tables - * @throws IOException exception - */ - public void addIncrementalBackupTableSet(Set tables) throws IOException { - systemTable.addIncrementalBackupTableSet(tables, backupInfo.getBackupRootDir()); - } - - /** - * Saves list of WAL files after incremental backup operation. These files will be stored until - * TTL expiration and are used by Backup Log Cleaner plug-in to determine which WAL files can be - * safely purged. - */ - public void recordWALFiles(List files) throws IOException { - systemTable.addWALFiles(files, backupInfo.getBackupId(), backupInfo.getBackupRootDir()); - } - - /** - * Get WAL files iterator - * @return WAL files iterator from backup system table - * @throws IOException - */ - public Iterator getWALFilesFromBackupSystem() throws IOException { - return systemTable.getWALFilesIterator(backupInfo.getBackupRootDir()); - } - - public Connection getConnection() { - return conn; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java deleted file mode 100644 index 4e6f6fff906..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java +++ /dev/null @@ -1,674 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; - -/** - * Backup manifest contains all the meta data of a backup image. The manifest info will be bundled - * as manifest file together with data. So that each backup image will contain all the info needed - * for restore. BackupManifest is a storage container for BackupImage. - * It is responsible for storing/reading backup image data and has some additional utility methods. - * - */ -@InterfaceAudience.Private -public class BackupManifest { - - private static final Log LOG = LogFactory.getLog(BackupManifest.class); - - // manifest file name - public static final String MANIFEST_FILE_NAME = ".backup.manifest"; - - /** - * Backup image, the dependency graph is made up by series of backup images BackupImage contains - * all the relevant information to restore the backup and is used during restore operation - */ - - public static class BackupImage implements Comparable { - - static class Builder { - BackupImage image; - - Builder() { - image = new BackupImage(); - } - - Builder withBackupId(String backupId) { - image.setBackupId(backupId); - return this; - } - - Builder withType(BackupType type) { - image.setType(type); - return this; - } - - Builder withRootDir(String rootDir) { - image.setRootDir(rootDir); - return this; - } - - Builder withTableList(List tableList) { - image.setTableList(tableList); - return this; - } - - Builder withStartTime(long startTime) { - image.setStartTs(startTime); - return this; - } - - Builder withCompleteTime(long completeTime) { - image.setCompleteTs(completeTime); - return this; - } - - BackupImage build() { - return image; - } - - } - - private String backupId; - private BackupType type; - private String rootDir; - private List tableList; - private long startTs; - private long completeTs; - private ArrayList ancestors; - private HashMap> incrTimeRanges; - - static Builder newBuilder() { - return new Builder(); - } - - public BackupImage() { - super(); - } - - private BackupImage(String backupId, BackupType type, String rootDir, - List tableList, long startTs, long completeTs) { - this.backupId = backupId; - this.type = type; - this.rootDir = rootDir; - this.tableList = tableList; - this.startTs = startTs; - this.completeTs = completeTs; - } - - static BackupImage fromProto(BackupProtos.BackupImage im) { - String backupId = im.getBackupId(); - String rootDir = im.getBackupRootDir(); - long startTs = im.getStartTs(); - long completeTs = im.getCompleteTs(); - List tableListList = im.getTableListList(); - List tableList = new ArrayList(); - for (HBaseProtos.TableName tn : tableListList) { - tableList.add(ProtobufUtil.toTableName(tn)); - } - - List ancestorList = im.getAncestorsList(); - - BackupType type = - im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL - : BackupType.INCREMENTAL; - - BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs); - for (BackupProtos.BackupImage img : ancestorList) { - image.addAncestor(fromProto(img)); - } - image.setIncrTimeRanges(loadIncrementalTimestampMap(im)); - return image; - } - - BackupProtos.BackupImage toProto() { - BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder(); - builder.setBackupId(backupId); - builder.setCompleteTs(completeTs); - builder.setStartTs(startTs); - builder.setBackupRootDir(rootDir); - if (type == BackupType.FULL) { - builder.setBackupType(BackupProtos.BackupType.FULL); - } else { - builder.setBackupType(BackupProtos.BackupType.INCREMENTAL); - } - - for (TableName name : tableList) { - builder.addTableList(ProtobufUtil.toProtoTableName(name)); - } - - if (ancestors != null) { - for (BackupImage im : ancestors) { - builder.addAncestors(im.toProto()); - } - } - - setIncrementalTimestampMap(builder); - return builder.build(); - } - - private static HashMap> loadIncrementalTimestampMap( - BackupProtos.BackupImage proto) { - List list = proto.getTstMapList(); - - HashMap> incrTimeRanges = - new HashMap>(); - if (list == null || list.size() == 0) return incrTimeRanges; - for (BackupProtos.TableServerTimestamp tst : list) { - TableName tn = ProtobufUtil.toTableName(tst.getTableName()); - HashMap map = incrTimeRanges.get(tn); - if (map == null) { - map = new HashMap(); - incrTimeRanges.put(tn, map); - } - List listSt = tst.getServerTimestampList(); - for (BackupProtos.ServerTimestamp stm : listSt) { - ServerName sn = ProtobufUtil.toServerName(stm.getServerName()); - map.put(sn.getHostname() + ":" + sn.getPort(), stm.getTimestamp()); - } - } - return incrTimeRanges; - } - - private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder) { - if (this.incrTimeRanges == null) { - return; - } - for (Entry> entry : this.incrTimeRanges.entrySet()) { - TableName key = entry.getKey(); - HashMap value = entry.getValue(); - BackupProtos.TableServerTimestamp.Builder tstBuilder = - BackupProtos.TableServerTimestamp.newBuilder(); - tstBuilder.setTableName(ProtobufUtil.toProtoTableName(key)); - - for (Map.Entry entry2 : value.entrySet()) { - String s = entry2.getKey(); - BackupProtos.ServerTimestamp.Builder stBuilder = - BackupProtos.ServerTimestamp.newBuilder(); - HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder(); - ServerName sn = ServerName.parseServerName(s); - snBuilder.setHostName(sn.getHostname()); - snBuilder.setPort(sn.getPort()); - stBuilder.setServerName(snBuilder.build()); - stBuilder.setTimestamp(entry2.getValue()); - tstBuilder.addServerTimestamp(stBuilder.build()); - } - builder.addTstMap(tstBuilder.build()); - } - } - - public String getBackupId() { - return backupId; - } - - private void setBackupId(String backupId) { - this.backupId = backupId; - } - - public BackupType getType() { - return type; - } - - private void setType(BackupType type) { - this.type = type; - } - - public String getRootDir() { - return rootDir; - } - - private void setRootDir(String rootDir) { - this.rootDir = rootDir; - } - - public List getTableNames() { - return tableList; - } - - private void setTableList(List tableList) { - this.tableList = tableList; - } - - public long getStartTs() { - return startTs; - } - - private void setStartTs(long startTs) { - this.startTs = startTs; - } - - public long getCompleteTs() { - return completeTs; - } - - private void setCompleteTs(long completeTs) { - this.completeTs = completeTs; - } - - public ArrayList getAncestors() { - if (this.ancestors == null) { - this.ancestors = new ArrayList(); - } - return this.ancestors; - } - - public void removeAncestors(List backupIds) { - List toRemove = new ArrayList(); - for (BackupImage im : this.ancestors) { - if (backupIds.contains(im.getBackupId())) { - toRemove.add(im); - } - } - this.ancestors.removeAll(toRemove); - } - - private void addAncestor(BackupImage backupImage) { - this.getAncestors().add(backupImage); - } - - public boolean hasAncestor(String token) { - for (BackupImage image : this.getAncestors()) { - if (image.getBackupId().equals(token)) { - return true; - } - } - return false; - } - - public boolean hasTable(TableName table) { - return tableList.contains(table); - } - - @Override - public int compareTo(BackupImage other) { - String thisBackupId = this.getBackupId(); - String otherBackupId = other.getBackupId(); - int index1 = thisBackupId.lastIndexOf("_"); - int index2 = otherBackupId.lastIndexOf("_"); - String name1 = thisBackupId.substring(0, index1); - String name2 = otherBackupId.substring(0, index2); - if (name1.equals(name2)) { - Long thisTS = Long.valueOf(thisBackupId.substring(index1 + 1)); - Long otherTS = Long.valueOf(otherBackupId.substring(index2 + 1)); - return thisTS.compareTo(otherTS); - } else { - return name1.compareTo(name2); - } - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof BackupImage) { - return this.compareTo((BackupImage) obj) == 0; - } - return false; - } - - @Override - public int hashCode() { - int hash = 33 * this.getBackupId().hashCode() + type.hashCode(); - hash = 33 * hash + rootDir.hashCode(); - hash = 33 * hash + Long.valueOf(startTs).hashCode(); - hash = 33 * hash + Long.valueOf(completeTs).hashCode(); - for (TableName table : tableList) { - hash = 33 * hash + table.hashCode(); - } - return hash; - } - - public HashMap> getIncrTimeRanges() { - return incrTimeRanges; - } - - private void setIncrTimeRanges(HashMap> incrTimeRanges) { - this.incrTimeRanges = incrTimeRanges; - } - } - - // backup image directory - private String tableBackupDir = null; - private BackupImage backupImage; - - /** - * Construct manifest for a ongoing backup. - * @param backup The ongoing backup info - */ - public BackupManifest(BackupInfo backup) { - - BackupImage.Builder builder = BackupImage.newBuilder(); - this.backupImage = - builder.withBackupId(backup.getBackupId()).withType(backup.getType()) - .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames()) - .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); - } - - /** - * Construct a table level manifest for a backup of the named table. - * @param backup The ongoing backup session info - */ - public BackupManifest(BackupInfo backup, TableName table) { - this.tableBackupDir = backup.getTableBackupDir(table); - List tables = new ArrayList(); - tables.add(table); - BackupImage.Builder builder = BackupImage.newBuilder(); - this.backupImage = - builder.withBackupId(backup.getBackupId()).withType(backup.getType()) - .withRootDir(backup.getBackupRootDir()).withTableList(tables) - .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); - } - - /** - * Construct manifest from a backup directory. - * @param conf configuration - * @param backupPath backup path - * @throws IOException - */ - - public BackupManifest(Configuration conf, Path backupPath) throws IOException { - this(backupPath.getFileSystem(conf), backupPath); - } - - /** - * Construct manifest from a backup directory. - * @param fs the FileSystem - * @param backupPath backup path - * @throws BackupException exception - */ - - public BackupManifest(FileSystem fs, Path backupPath) throws BackupException { - if (LOG.isDebugEnabled()) { - LOG.debug("Loading manifest from: " + backupPath.toString()); - } - // The input backupDir may not exactly be the backup table dir. - // It could be the backup log dir where there is also a manifest file stored. - // This variable's purpose is to keep the correct and original location so - // that we can store/persist it. - try { - - FileStatus[] subFiles = BackupUtils.listStatus(fs, backupPath, null); - if (subFiles == null) { - String errorMsg = backupPath.toString() + " does not exist"; - LOG.error(errorMsg); - throw new IOException(errorMsg); - } - for (FileStatus subFile : subFiles) { - if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) { - - // load and set manifest field from file content - FSDataInputStream in = fs.open(subFile.getPath()); - long len = subFile.getLen(); - byte[] pbBytes = new byte[(int) len]; - in.readFully(pbBytes); - BackupProtos.BackupImage proto = null; - try { - proto = BackupProtos.BackupImage.parseFrom(pbBytes); - } catch (Exception e) { - throw new BackupException(e); - } - this.backupImage = BackupImage.fromProto(proto); - LOG.debug("Loaded manifest instance from manifest file: " - + BackupUtils.getPath(subFile.getPath())); - return; - } - } - String errorMsg = "No manifest file found in: " + backupPath.toString(); - throw new IOException(errorMsg); - - } catch (IOException e) { - throw new BackupException(e.getMessage()); - } - } - - public BackupType getType() { - return backupImage.getType(); - } - - /** - * Get the table set of this image. - * @return The table set list - */ - public List getTableList() { - return backupImage.getTableNames(); - } - - /** - * TODO: fix it. Persist the manifest file. - * @throws IOException IOException when storing the manifest file. - */ - - public void store(Configuration conf) throws BackupException { - byte[] data = backupImage.toProto().toByteArray(); - // write the file, overwrite if already exist - Path manifestFilePath = - new Path(HBackupFileSystem.getBackupPath(backupImage.getRootDir(), - backupImage.getBackupId()), MANIFEST_FILE_NAME); - try (FSDataOutputStream out = - manifestFilePath.getFileSystem(conf).create(manifestFilePath, true);) { - out.write(data); - } catch (IOException e) { - throw new BackupException(e.getMessage()); - } - - LOG.info("Manifest file stored to " + manifestFilePath); - } - - /** - * Get this backup image. - * @return the backup image. - */ - public BackupImage getBackupImage() { - return backupImage; - } - - /** - * Add dependent backup image for this backup. - * @param image The direct dependent backup image - */ - public void addDependentImage(BackupImage image) { - this.backupImage.addAncestor(image); - } - - /** - * Set the incremental timestamp map directly. - * @param incrTimestampMap timestamp map - */ - public void setIncrTimestampMap(HashMap> incrTimestampMap) { - this.backupImage.setIncrTimeRanges(incrTimestampMap); - } - - public Map> getIncrTimestampMap() { - return backupImage.getIncrTimeRanges(); - } - - /** - * Get the image list of this backup for restore in time order. - * @param reverse If true, then output in reverse order, otherwise in time order from old to new - * @return the backup image list for restore in time order - */ - public ArrayList getRestoreDependentList(boolean reverse) { - TreeMap restoreImages = new TreeMap(); - restoreImages.put(backupImage.startTs, backupImage); - for (BackupImage image : backupImage.getAncestors()) { - restoreImages.put(Long.valueOf(image.startTs), image); - } - return new ArrayList(reverse ? (restoreImages.descendingMap().values()) - : (restoreImages.values())); - } - - /** - * Get the dependent image list for a specific table of this backup in time order from old to new - * if want to restore to this backup image level. - * @param table table - * @return the backup image list for a table in time order - */ - public ArrayList getDependentListByTable(TableName table) { - ArrayList tableImageList = new ArrayList(); - ArrayList imageList = getRestoreDependentList(true); - for (BackupImage image : imageList) { - if (image.hasTable(table)) { - tableImageList.add(image); - if (image.getType() == BackupType.FULL) { - break; - } - } - } - Collections.reverse(tableImageList); - return tableImageList; - } - - /** - * Get the full dependent image list in the whole dependency scope for a specific table of this - * backup in time order from old to new. - * @param table table - * @return the full backup image list for a table in time order in the whole scope of the - * dependency of this image - */ - public ArrayList getAllDependentListByTable(TableName table) { - ArrayList tableImageList = new ArrayList(); - ArrayList imageList = getRestoreDependentList(false); - for (BackupImage image : imageList) { - if (image.hasTable(table)) { - tableImageList.add(image); - } - } - return tableImageList; - } - - /** - * Check whether backup image1 could cover backup image2 or not. - * @param image1 backup image 1 - * @param image2 backup image 2 - * @return true if image1 can cover image2, otherwise false - */ - public static boolean canCoverImage(BackupImage image1, BackupImage image2) { - // image1 can cover image2 only when the following conditions are satisfied: - // - image1 must not be an incremental image; - // - image1 must be taken after image2 has been taken; - // - table set of image1 must cover the table set of image2. - if (image1.getType() == BackupType.INCREMENTAL) { - return false; - } - if (image1.getStartTs() < image2.getStartTs()) { - return false; - } - List image1TableList = image1.getTableNames(); - List image2TableList = image2.getTableNames(); - boolean found = false; - for (int i = 0; i < image2TableList.size(); i++) { - found = false; - for (int j = 0; j < image1TableList.size(); j++) { - if (image2TableList.get(i).equals(image1TableList.get(j))) { - found = true; - break; - } - } - if (!found) { - return false; - } - } - - LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId()); - return true; - } - - /** - * Check whether backup image set could cover a backup image or not. - * @param fullImages The backup image set - * @param image The target backup image - * @return true if fullImages can cover image, otherwise false - */ - public static boolean canCoverImage(ArrayList fullImages, BackupImage image) { - // fullImages can cover image only when the following conditions are satisfied: - // - each image of fullImages must not be an incremental image; - // - each image of fullImages must be taken after image has been taken; - // - sum table set of fullImages must cover the table set of image. - for (BackupImage image1 : fullImages) { - if (image1.getType() == BackupType.INCREMENTAL) { - return false; - } - if (image1.getStartTs() < image.getStartTs()) { - return false; - } - } - - ArrayList image1TableList = new ArrayList(); - for (BackupImage image1 : fullImages) { - List tableList = image1.getTableNames(); - for (TableName table : tableList) { - image1TableList.add(table.getNameAsString()); - } - } - ArrayList image2TableList = new ArrayList(); - List tableList = image.getTableNames(); - for (TableName table : tableList) { - image2TableList.add(table.getNameAsString()); - } - - for (int i = 0; i < image2TableList.size(); i++) { - if (image1TableList.contains(image2TableList.get(i)) == false) { - return false; - } - } - - LOG.debug("Full image set can cover image " + image.getBackupId()); - return true; - } - - public BackupInfo toBackupInfo() { - BackupInfo info = new BackupInfo(); - info.setType(backupImage.getType()); - List list = backupImage.getTableNames(); - TableName[] tables = new TableName[list.size()]; - info.addTables(list.toArray(tables)); - info.setBackupId(backupImage.getBackupId()); - info.setStartTs(backupImage.getStartTs()); - info.setBackupRootDir(backupImage.getRootDir()); - if (backupImage.getType() == BackupType.INCREMENTAL) { - info.setHLogTargetDir(BackupUtils.getLogBackupDir(backupImage.getRootDir(), - backupImage.getBackupId())); - } - return info; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java deleted file mode 100644 index de3ccc759c3..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ /dev/null @@ -1,2050 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.impl; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.Pair; - -/** - * This class provides API to access backup system table
- * - * Backup system table schema:
- *

    - *
  • 1. Backup sessions rowkey= "session:"+backupId; value =serialized BackupInfo
    • - *
  • 2. Backup start code rowkey = "startcode:"+backupRoot; value = startcode
    • - *
  • 3. Incremental backup set rowkey="incrbackupset:"+backupRoot; value=[list of tables]
    • - *
  • 4. Table-RS-timestamp map rowkey="trslm:"+backupRoot+table_name; - * value = map[RS-> last WAL timestamp]
    • - *
  • 5. RS - WAL ts map rowkey="rslogts:"+backupRoot +server; value = last WAL timestamp
    • - *
  • 6. WALs recorded rowkey="wals:"+WAL unique file name; - * value = backupId and full WAL file name
    • - *

- */ - -@InterfaceAudience.Private -public final class BackupSystemTable implements Closeable { - private static final Log LOG = LogFactory.getLog(BackupSystemTable.class); - - static class WALItem { - String backupId; - String walFile; - String backupRoot; - - WALItem(String backupId, String walFile, String backupRoot) { - this.backupId = backupId; - this.walFile = walFile; - this.backupRoot = backupRoot; - } - - public String getBackupId() { - return backupId; - } - - public String getWalFile() { - return walFile; - } - - public String getBackupRoot() { - return backupRoot; - } - - @Override - public String toString() { - return Path.SEPARATOR + backupRoot + Path.SEPARATOR + backupId + Path.SEPARATOR + walFile; - } - - } - - private TableName tableName; - /** - * Stores backup sessions (contexts) - */ - final static byte[] SESSIONS_FAMILY = "session".getBytes(); - /** - * Stores other meta - */ - final static byte[] META_FAMILY = "meta".getBytes(); - final static byte[] BULK_LOAD_FAMILY = "bulk".getBytes(); - /** - * Connection to HBase cluster, shared among all instances - */ - private final Connection connection; - - private final static String BACKUP_INFO_PREFIX = "session:"; - private final static String START_CODE_ROW = "startcode:"; - private final static byte[] ACTIVE_SESSION_ROW = "activesession:".getBytes(); - private final static byte[] ACTIVE_SESSION_COL = "c".getBytes(); - - private final static byte[] ACTIVE_SESSION_YES = "yes".getBytes(); - private final static byte[] ACTIVE_SESSION_NO = "no".getBytes(); - - private final static String INCR_BACKUP_SET = "incrbackupset:"; - private final static String TABLE_RS_LOG_MAP_PREFIX = "trslm:"; - private final static String RS_LOG_TS_PREFIX = "rslogts:"; - - private final static String BULK_LOAD_PREFIX = "bulk:"; - private final static byte[] BULK_LOAD_PREFIX_BYTES = BULK_LOAD_PREFIX.getBytes(); - private final static byte[] DELETE_OP_ROW = "delete_op_row".getBytes(); - private final static byte[] MERGE_OP_ROW = "merge_op_row".getBytes(); - - final static byte[] TBL_COL = Bytes.toBytes("tbl"); - final static byte[] FAM_COL = Bytes.toBytes("fam"); - final static byte[] PATH_COL = Bytes.toBytes("path"); - final static byte[] STATE_COL = Bytes.toBytes("state"); - // the two states a bulk loaded file can be - final static byte[] BL_PREPARE = Bytes.toBytes("R"); - final static byte[] BL_COMMIT = Bytes.toBytes("D"); - - private final static String WALS_PREFIX = "wals:"; - private final static String SET_KEY_PREFIX = "backupset:"; - - // separator between BULK_LOAD_PREFIX and ordinals - protected final static String BLK_LD_DELIM = ":"; - private final static byte[] EMPTY_VALUE = new byte[] {}; - - // Safe delimiter in a string - private final static String NULL = "\u0000"; - - public BackupSystemTable(Connection conn) throws IOException { - this.connection = conn; - tableName = BackupSystemTable.getTableName(conn.getConfiguration()); - checkSystemTable(); - } - - private void checkSystemTable() throws IOException { - try (Admin admin = connection.getAdmin();) { - - verifyNamespaceExists(admin); - - if (!admin.tableExists(tableName)) { - HTableDescriptor backupHTD = - BackupSystemTable.getSystemTableDescriptor(connection.getConfiguration()); - admin.createTable(backupHTD); - } - waitForSystemTable(admin); - } - } - - private void verifyNamespaceExists(Admin admin) throws IOException { - String namespaceName = tableName.getNamespaceAsString(); - NamespaceDescriptor ns = NamespaceDescriptor.create(namespaceName).build(); - NamespaceDescriptor[] list = admin.listNamespaceDescriptors(); - boolean exists = false; - for (NamespaceDescriptor nsd : list) { - if (nsd.getName().equals(ns.getName())) { - exists = true; - break; - } - } - if (!exists) { - admin.createNamespace(ns); - } - } - - private void waitForSystemTable(Admin admin) throws IOException { - long TIMEOUT = 60000; - long startTime = EnvironmentEdgeManager.currentTime(); - while (!admin.tableExists(tableName) || !admin.isTableAvailable(tableName)) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - } - if (EnvironmentEdgeManager.currentTime() - startTime > TIMEOUT) { - throw new IOException("Failed to create backup system table after " + TIMEOUT + "ms"); - } - } - LOG.debug("Backup table exists and available"); - - } - - @Override - public void close() { - // do nothing - } - - /** - * Updates status (state) of a backup session in backup system table table - * @param info backup info - * @throws IOException exception - */ - public void updateBackupInfo(BackupInfo info) throws IOException { - - if (LOG.isTraceEnabled()) { - LOG.trace("update backup status in backup system table for: " + info.getBackupId() - + " set status=" + info.getState()); - } - try (Table table = connection.getTable(tableName)) { - Put put = createPutForBackupInfo(info); - table.put(put); - } - } - - /* - * @param backupId the backup Id - * @return Map of rows to path of bulk loaded hfile - */ - Map readBulkLoadedFiles(String backupId) throws IOException { - Scan scan = BackupSystemTable.createScanForBulkLoadedFiles(backupId); - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - Map map = new TreeMap<>(Bytes.BYTES_COMPARATOR); - while ((res = scanner.next()) != null) { - res.advance(); - byte[] row = CellUtil.cloneRow(res.listCells().get(0)); - for (Cell cell : res.listCells()) { - if (CellUtil.compareQualifiers(cell, BackupSystemTable.PATH_COL, 0, - BackupSystemTable.PATH_COL.length) == 0) { - map.put(row, Bytes.toString(CellUtil.cloneValue(cell))); - } - } - } - return map; - } - } - - /* - * Used during restore - * @param backupId the backup Id - * @param sTableList List of tables - * @return array of Map of family to List of Paths - */ - public Map>[] readBulkLoadedFiles(String backupId, List sTableList) - throws IOException { - Scan scan = BackupSystemTable.createScanForBulkLoadedFiles(backupId); - Map>[] mapForSrc = new Map[sTableList == null ? 1 : sTableList.size()]; - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - TableName tbl = null; - byte[] fam = null; - String path = null; - for (Cell cell : res.listCells()) { - if (CellUtil.compareQualifiers(cell, BackupSystemTable.TBL_COL, 0, - BackupSystemTable.TBL_COL.length) == 0) { - tbl = TableName.valueOf(CellUtil.cloneValue(cell)); - } else if (CellUtil.compareQualifiers(cell, BackupSystemTable.FAM_COL, 0, - BackupSystemTable.FAM_COL.length) == 0) { - fam = CellUtil.cloneValue(cell); - } else if (CellUtil.compareQualifiers(cell, BackupSystemTable.PATH_COL, 0, - BackupSystemTable.PATH_COL.length) == 0) { - path = Bytes.toString(CellUtil.cloneValue(cell)); - } - } - int srcIdx = IncrementalTableBackupClient.getIndex(tbl, sTableList); - if (srcIdx == -1) { - // the table is not among the query - continue; - } - if (mapForSrc[srcIdx] == null) { - mapForSrc[srcIdx] = new TreeMap<>(Bytes.BYTES_COMPARATOR); - } - List files; - if (!mapForSrc[srcIdx].containsKey(fam)) { - files = new ArrayList(); - mapForSrc[srcIdx].put(fam, files); - } else { - files = mapForSrc[srcIdx].get(fam); - } - files.add(new Path(path)); - if (LOG.isDebugEnabled()) { - LOG.debug("found bulk loaded file : " + tbl + " " + Bytes.toString(fam) + " " + path); - } - } - ; - return mapForSrc; - } - } - - /* - * @param map Map of row keys to path of bulk loaded hfile - */ - void deleteBulkLoadedFiles(Map map) throws IOException { - try (Table table = connection.getTable(tableName)) { - List dels = new ArrayList<>(); - for (byte[] row : map.keySet()) { - dels.add(new Delete(row).addFamily(BackupSystemTable.META_FAMILY)); - } - table.delete(dels); - } - } - - /** - * Deletes backup status from backup system table table - * @param backupId backup id - * @throws IOException exception - */ - - public void deleteBackupInfo(String backupId) throws IOException { - - if (LOG.isTraceEnabled()) { - LOG.trace("delete backup status in backup system table for " + backupId); - } - try (Table table = connection.getTable(tableName)) { - Delete del = createDeleteForBackupInfo(backupId); - table.delete(del); - } - } - - /* - * For postBulkLoadHFile() hook. - * @param tabName table name - * @param region the region receiving hfile - * @param finalPaths family and associated hfiles - */ - public void writePathsPostBulkLoad(TableName tabName, byte[] region, - Map> finalPaths) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("write bulk load descriptor to backup " + tabName + " with " + finalPaths.size() - + " entries"); - } - try (Table table = connection.getTable(tableName)) { - List puts = BackupSystemTable.createPutForCommittedBulkload(tabName, region, finalPaths); - table.put(puts); - LOG.debug("written " + puts.size() + " rows for bulk load of " + tabName); - } - } - - /* - * For preCommitStoreFile() hook - * @param tabName table name - * @param region the region receiving hfile - * @param family column family - * @param pairs list of paths for hfiles - */ - public void writeFilesForBulkLoadPreCommit(TableName tabName, byte[] region, final byte[] family, - final List> pairs) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("write bulk load descriptor to backup " + tabName + " with " + pairs.size() - + " entries"); - } - try (Table table = connection.getTable(tableName)) { - List puts = - BackupSystemTable.createPutForPreparedBulkload(tabName, region, family, pairs); - table.put(puts); - LOG.debug("written " + puts.size() + " rows for bulk load of " + tabName); - } - } - - /* - * Removes rows recording bulk loaded hfiles from backup table - * @param lst list of table names - * @param rows the rows to be deleted - */ - public void removeBulkLoadedRows(List lst, List rows) throws IOException { - try (Table table = connection.getTable(tableName)) { - List lstDels = new ArrayList<>(); - for (byte[] row : rows) { - Delete del = new Delete(row); - lstDels.add(del); - LOG.debug("orig deleting the row: " + Bytes.toString(row)); - } - table.delete(lstDels); - LOG.debug("deleted " + rows.size() + " original bulkload rows for " + lst.size() + " tables"); - } - } - - /* - * Reads the rows from backup table recording bulk loaded hfiles - * @param tableList list of table names - * @return The keys of the Map are table, region and column family. Value of the map reflects - * whether the hfile was recorded by preCommitStoreFile hook (true) - */ - public Pair>>>>, List> - readBulkloadRows(List tableList) throws IOException { - Map>>>> map = new HashMap<>(); - List rows = new ArrayList<>(); - for (TableName tTable : tableList) { - Scan scan = BackupSystemTable.createScanForOrigBulkLoadedFiles(tTable); - Map>>> tblMap = map.get(tTable); - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - String fam = null; - String path = null; - boolean raw = false; - byte[] row = null; - String region = null; - for (Cell cell : res.listCells()) { - row = CellUtil.cloneRow(cell); - rows.add(row); - String rowStr = Bytes.toString(row); - region = BackupSystemTable.getRegionNameFromOrigBulkLoadRow(rowStr); - if (CellUtil.compareQualifiers(cell, BackupSystemTable.FAM_COL, 0, - BackupSystemTable.FAM_COL.length) == 0) { - fam = Bytes.toString(CellUtil.cloneValue(cell)); - } else if (CellUtil.compareQualifiers(cell, BackupSystemTable.PATH_COL, 0, - BackupSystemTable.PATH_COL.length) == 0) { - path = Bytes.toString(CellUtil.cloneValue(cell)); - } else if (CellUtil.compareQualifiers(cell, BackupSystemTable.STATE_COL, 0, - BackupSystemTable.STATE_COL.length) == 0) { - byte[] state = CellUtil.cloneValue(cell); - if (Bytes.equals(BackupSystemTable.BL_PREPARE, state)) { - raw = true; - } else raw = false; - } - } - if (map.get(tTable) == null) { - map.put(tTable, new HashMap>>>()); - tblMap = map.get(tTable); - } - if (tblMap.get(region) == null) { - tblMap.put(region, new HashMap>>()); - } - Map>> famMap = tblMap.get(region); - if (famMap.get(fam) == null) { - famMap.put(fam, new ArrayList>()); - } - famMap.get(fam).add(new Pair<>(path, raw)); - LOG.debug("found orig " + path + " for " + fam + " of table " + region); - } - } - } - return new Pair<>(map, rows); - } - - /* - * @param sTableList List of tables - * @param maps array of Map of family to List of Paths - * @param backupId the backup Id - */ - public void writeBulkLoadedFiles(List sTableList, Map>[] maps, - String backupId) throws IOException { - try (Table table = connection.getTable(tableName)) { - long ts = EnvironmentEdgeManager.currentTime(); - int cnt = 0; - List puts = new ArrayList<>(); - for (int idx = 0; idx < maps.length; idx++) { - Map> map = maps[idx]; - TableName tn = sTableList.get(idx); - if (map == null) continue; - for (Map.Entry> entry : map.entrySet()) { - byte[] fam = entry.getKey(); - List paths = entry.getValue(); - for (Path p : paths) { - Put put = - BackupSystemTable.createPutForBulkLoadedFile(tn, fam, p.toString(), backupId, ts, - cnt++); - puts.add(put); - } - } - } - if (!puts.isEmpty()) { - table.put(puts); - } - } - } - - /** - * Reads backup status object (instance of backup info) from backup system table table - * @param backupId backup id - * @return Current status of backup session or null - */ - - public BackupInfo readBackupInfo(String backupId) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("read backup status from backup system table for: " + backupId); - } - - try (Table table = connection.getTable(tableName)) { - Get get = createGetForBackupInfo(backupId); - Result res = table.get(get); - if (res.isEmpty()) { - return null; - } - return resultToBackupInfo(res); - } - } - - /** - * Read the last backup start code (timestamp) of last successful backup. Will return null if - * there is no start code stored on hbase or the value is of length 0. These two cases indicate - * there is no successful backup completed so far. - * @param backupRoot directory path to backup destination - * @return the timestamp of last successful backup - * @throws IOException exception - */ - public String readBackupStartCode(String backupRoot) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("read backup start code from backup system table"); - } - try (Table table = connection.getTable(tableName)) { - Get get = createGetForStartCode(backupRoot); - Result res = table.get(get); - if (res.isEmpty()) { - return null; - } - Cell cell = res.listCells().get(0); - byte[] val = CellUtil.cloneValue(cell); - if (val.length == 0) { - return null; - } - return new String(val); - } - } - - /** - * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte. - * @param startCode start code - * @param backupRoot root directory path to backup - * @throws IOException exception - */ - public void writeBackupStartCode(Long startCode, String backupRoot) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("write backup start code to backup system table " + startCode); - } - try (Table table = connection.getTable(tableName)) { - Put put = createPutForStartCode(startCode.toString(), backupRoot); - table.put(put); - } - } - - /** - * Exclusive operations are: - * create, delete, merge - * @throws IOException - */ - public void startBackupExclusiveOperation() throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Start new backup exclusive operation"); - } - try (Table table = connection.getTable(tableName)) { - Put put = createPutForStartBackupSession(); - // First try to put if row does not exist - if (!table.checkAndPut(ACTIVE_SESSION_ROW, SESSIONS_FAMILY, ACTIVE_SESSION_COL, null, put)) { - // Row exists, try to put if value == ACTIVE_SESSION_NO - if (!table.checkAndPut(ACTIVE_SESSION_ROW, SESSIONS_FAMILY, ACTIVE_SESSION_COL, - ACTIVE_SESSION_NO, put)) { - throw new IOException("There is an active backup exclusive operation"); - } - } - } - } - - private Put createPutForStartBackupSession() { - Put put = new Put(ACTIVE_SESSION_ROW); - put.addColumn(SESSIONS_FAMILY, ACTIVE_SESSION_COL, ACTIVE_SESSION_YES); - return put; - } - - public void finishBackupExclusiveOperation() throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Finish backup exclusive operation"); - } - try (Table table = connection.getTable(tableName)) { - Put put = createPutForStopBackupSession(); - if (!table.checkAndPut(ACTIVE_SESSION_ROW, SESSIONS_FAMILY, ACTIVE_SESSION_COL, - ACTIVE_SESSION_YES, put)) { - throw new IOException("There is no active backup exclusive operation"); - } - } - } - - private Put createPutForStopBackupSession() { - Put put = new Put(ACTIVE_SESSION_ROW); - put.addColumn(SESSIONS_FAMILY, ACTIVE_SESSION_COL, ACTIVE_SESSION_NO); - return put; - } - - /** - * Get the Region Servers log information after the last log roll from backup system table. - * @param backupRoot root directory path to backup - * @return RS log info - * @throws IOException exception - */ - public HashMap readRegionServerLastLogRollResult(String backupRoot) - throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("read region server last roll log result to backup system table"); - } - - Scan scan = createScanForReadRegionServerLastLogRollResult(backupRoot); - - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - HashMap rsTimestampMap = new HashMap(); - while ((res = scanner.next()) != null) { - res.advance(); - Cell cell = res.current(); - byte[] row = CellUtil.cloneRow(cell); - String server = getServerNameForReadRegionServerLastLogRollResult(row); - byte[] data = CellUtil.cloneValue(cell); - rsTimestampMap.put(server, Bytes.toLong(data)); - } - return rsTimestampMap; - } - } - - /** - * Writes Region Server last roll log result (timestamp) to backup system table table - * @param server Region Server name - * @param ts last log timestamp - * @param backupRoot root directory path to backup - * @throws IOException exception - */ - public void writeRegionServerLastLogRollResult(String server, Long ts, String backupRoot) - throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("write region server last roll log result to backup system table"); - } - try (Table table = connection.getTable(tableName)) { - Put put = createPutForRegionServerLastLogRollResult(server, ts, backupRoot); - table.put(put); - } - } - - /** - * Get all completed backup information (in desc order by time) - * @param onlyCompleted true, if only successfully completed sessions - * @return history info of BackupCompleteData - * @throws IOException exception - */ - public ArrayList getBackupHistory(boolean onlyCompleted) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("get backup history from backup system table"); - } - ArrayList list; - BackupState state = onlyCompleted ? BackupState.COMPLETE : BackupState.ANY; - list = getBackupInfos(state); - return BackupUtils.sortHistoryListDesc(list); - } - - /** - * Get all backups history - * @return list of backup info - * @throws IOException - */ - public List getBackupHistory() throws IOException { - return getBackupHistory(false); - } - - /** - * Get first n backup history records - * @param n number of records, if n== -1 - max number - * is ignored - * @return list of records - * @throws IOException - */ - public List getHistory(int n) throws IOException { - - List history = getBackupHistory(); - if (n == -1 || history.size() <= n) return history; - List list = new ArrayList(); - for (int i = 0; i < n; i++) { - list.add(history.get(i)); - } - return list; - - } - - /** - * Get backup history records filtered by list of filters. - * @param n max number of records, if n == -1 , then max number - * is ignored - * @param filters list of filters - * @return backup records - * @throws IOException - */ - public List getBackupHistory(int n, BackupInfo.Filter... filters) throws IOException { - if (filters.length == 0) return getHistory(n); - - List history = getBackupHistory(); - List result = new ArrayList(); - for (BackupInfo bi : history) { - if (n >= 0 && result.size() == n) break; - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(bi)) { - passed = false; - break; - } - } - if (passed) { - result.add(bi); - } - } - return result; - - } - - /* - * Retrieve TableName's for completed backup of given type - * @param type backup type - * @return List of table names - */ - public List getTablesForBackupType(BackupType type) throws IOException { - Set names = new HashSet<>(); - List infos = getBackupHistory(true); - for (BackupInfo info : infos) { - if (info.getType() != type) continue; - names.addAll(info.getTableNames()); - } - return new ArrayList(names); - } - - /** - * Get history for backup destination - * @param backupRoot backup destination path - * @return List of backup info - * @throws IOException - */ - public List getBackupHistory(String backupRoot) throws IOException { - ArrayList history = getBackupHistory(false); - for (Iterator iterator = history.iterator(); iterator.hasNext();) { - BackupInfo info = iterator.next(); - if (!backupRoot.equals(info.getBackupRootDir())) { - iterator.remove(); - } - } - return history; - } - - /** - * Get history for a table - * @param name table name - * @return history for a table - * @throws IOException - */ - public List getBackupHistoryForTable(TableName name) throws IOException { - List history = getBackupHistory(); - List tableHistory = new ArrayList(); - for (BackupInfo info : history) { - List tables = info.getTableNames(); - if (tables.contains(name)) { - tableHistory.add(info); - } - } - return tableHistory; - } - - public Map> getBackupHistoryForTableSet(Set set, - String backupRoot) throws IOException { - List history = getBackupHistory(backupRoot); - Map> tableHistoryMap = - new HashMap>(); - for (Iterator iterator = history.iterator(); iterator.hasNext();) { - BackupInfo info = iterator.next(); - if (!backupRoot.equals(info.getBackupRootDir())) { - continue; - } - List tables = info.getTableNames(); - for (TableName tableName : tables) { - if (set.contains(tableName)) { - ArrayList list = tableHistoryMap.get(tableName); - if (list == null) { - list = new ArrayList(); - tableHistoryMap.put(tableName, list); - } - list.add(info); - } - } - } - return tableHistoryMap; - } - - /** - * Get all backup sessions with a given state (in descending order by time) - * @param state backup session state - * @return history info of backup info objects - * @throws IOException exception - */ - public ArrayList getBackupInfos(BackupState state) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("get backup infos from backup system table"); - } - - Scan scan = createScanForBackupHistory(); - ArrayList list = new ArrayList(); - - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - BackupInfo context = cellToBackupInfo(res.current()); - if (state != BackupState.ANY && context.getState() != state) { - continue; - } - list.add(context); - } - return list; - } - } - - /** - * Write the current timestamps for each regionserver to backup system table after a successful - * full or incremental backup. The saved timestamp is of the last log file that was backed up - * already. - * @param tables tables - * @param newTimestamps timestamps - * @param backupRoot root directory path to backup - * @throws IOException exception - */ - public void writeRegionServerLogTimestamp(Set tables, - HashMap newTimestamps, String backupRoot) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("write RS log time stamps to backup system table for tables [" - + StringUtils.join(tables, ",") + "]"); - } - List puts = new ArrayList(); - for (TableName table : tables) { - byte[] smapData = toTableServerTimestampProto(table, newTimestamps).toByteArray(); - Put put = createPutForWriteRegionServerLogTimestamp(table, smapData, backupRoot); - puts.add(put); - } - try (Table table = connection.getTable(tableName)) { - table.put(puts); - } - } - - /** - * Read the timestamp for each region server log after the last successful backup. Each table has - * its own set of the timestamps. The info is stored for each table as a concatenated string of - * rs->timestapmp - * @param backupRoot root directory path to backup - * @return the timestamp for each region server. key: tableName value: - * RegionServer,PreviousTimeStamp - * @throws IOException exception - */ - public HashMap> readLogTimestampMap(String backupRoot) - throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("read RS log ts from backup system table for root=" + backupRoot); - } - - HashMap> tableTimestampMap = - new HashMap>(); - - Scan scan = createScanForReadLogTimestampMap(backupRoot); - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - Cell cell = res.current(); - byte[] row = CellUtil.cloneRow(cell); - String tabName = getTableNameForReadLogTimestampMap(row); - TableName tn = TableName.valueOf(tabName); - byte[] data = CellUtil.cloneValue(cell); - if (data == null) { - throw new IOException("Data of last backup data from backup system table " - + "is empty. Create a backup first."); - } - if (data != null && data.length > 0) { - HashMap lastBackup = - fromTableServerTimestampProto(BackupProtos.TableServerTimestamp.parseFrom(data)); - tableTimestampMap.put(tn, lastBackup); - } - } - return tableTimestampMap; - } - } - - private BackupProtos.TableServerTimestamp toTableServerTimestampProto(TableName table, - Map map) { - BackupProtos.TableServerTimestamp.Builder tstBuilder = - BackupProtos.TableServerTimestamp.newBuilder(); - tstBuilder.setTableName(org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil - .toProtoTableName(table)); - - for (Entry entry : map.entrySet()) { - BackupProtos.ServerTimestamp.Builder builder = BackupProtos.ServerTimestamp.newBuilder(); - HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder(); - ServerName sn = ServerName.parseServerName(entry.getKey()); - snBuilder.setHostName(sn.getHostname()); - snBuilder.setPort(sn.getPort()); - builder.setServerName(snBuilder.build()); - builder.setTimestamp(entry.getValue()); - tstBuilder.addServerTimestamp(builder.build()); - } - - return tstBuilder.build(); - } - - private HashMap fromTableServerTimestampProto( - BackupProtos.TableServerTimestamp proto) { - HashMap map = new HashMap(); - List list = proto.getServerTimestampList(); - for (BackupProtos.ServerTimestamp st : list) { - ServerName sn = - org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toServerName(st.getServerName()); - map.put(sn.getHostname() + ":" + sn.getPort(), st.getTimestamp()); - } - return map; - } - - /** - * Return the current tables covered by incremental backup. - * @param backupRoot root directory path to backup - * @return set of tableNames - * @throws IOException exception - */ - public Set getIncrementalBackupTableSet(String backupRoot) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("get incremental backup table set from backup system table"); - } - TreeSet set = new TreeSet<>(); - - try (Table table = connection.getTable(tableName)) { - Get get = createGetForIncrBackupTableSet(backupRoot); - Result res = table.get(get); - if (res.isEmpty()) { - return set; - } - List cells = res.listCells(); - for (Cell cell : cells) { - // qualifier = table name - we use table names as qualifiers - set.add(TableName.valueOf(CellUtil.cloneQualifier(cell))); - } - return set; - } - } - - /** - * Add tables to global incremental backup set - * @param tables set of tables - * @param backupRoot root directory path to backup - * @throws IOException exception - */ - public void addIncrementalBackupTableSet(Set tables, String backupRoot) - throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Add incremental backup table set to backup system table. ROOT=" + backupRoot - + " tables [" + StringUtils.join(tables, " ") + "]"); - for (TableName table : tables) { - LOG.debug(table); - } - } - try (Table table = connection.getTable(tableName)) { - Put put = createPutForIncrBackupTableSet(tables, backupRoot); - table.put(put); - } - } - - /** - * Deletes incremental backup set for a backup destination - * @param backupRoot backup root - */ - - public void deleteIncrementalBackupTableSet(String backupRoot) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Delete incremental backup table set to backup system table. ROOT=" + backupRoot); - } - try (Table table = connection.getTable(tableName)) { - Delete delete = createDeleteForIncrBackupTableSet(backupRoot); - table.delete(delete); - } - } - - /** - * Register WAL files as eligible for deletion - * @param files files - * @param backupId backup id - * @param backupRoot root directory path to backup destination - * @throws IOException exception - */ - public void addWALFiles(List files, String backupId, String backupRoot) - throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("add WAL files to backup system table: " + backupId + " " + backupRoot + " files [" - + StringUtils.join(files, ",") + "]"); - for (String f : files) { - LOG.debug("add :" + f); - } - } - try (Table table = connection.getTable(tableName)) { - List puts = createPutsForAddWALFiles(files, backupId, backupRoot); - table.put(puts); - } - } - - /** - * Register WAL files as eligible for deletion - * @param backupRoot root directory path to backup - * @throws IOException exception - */ - public Iterator getWALFilesIterator(String backupRoot) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("get WAL files from backup system table"); - } - final Table table = connection.getTable(tableName); - Scan scan = createScanForGetWALs(backupRoot); - final ResultScanner scanner = table.getScanner(scan); - final Iterator it = scanner.iterator(); - return new Iterator() { - - @Override - public boolean hasNext() { - boolean next = it.hasNext(); - if (!next) { - // close all - try { - scanner.close(); - table.close(); - } catch (IOException e) { - LOG.error("Close WAL Iterator", e); - } - } - return next; - } - - @Override - public WALItem next() { - Result next = it.next(); - List cells = next.listCells(); - byte[] buf = cells.get(0).getValueArray(); - int len = cells.get(0).getValueLength(); - int offset = cells.get(0).getValueOffset(); - String backupId = new String(buf, offset, len); - buf = cells.get(1).getValueArray(); - len = cells.get(1).getValueLength(); - offset = cells.get(1).getValueOffset(); - String walFile = new String(buf, offset, len); - buf = cells.get(2).getValueArray(); - len = cells.get(2).getValueLength(); - offset = cells.get(2).getValueOffset(); - String backupRoot = new String(buf, offset, len); - return new WALItem(backupId, walFile, backupRoot); - } - - @Override - public void remove() { - // not implemented - throw new RuntimeException("remove is not supported"); - } - }; - - } - - /** - * Check if WAL file is eligible for deletion Future: to support all backup destinations - * @param file name of a file to check - * @return true, if deletable, false otherwise. - * @throws IOException exception - * TODO: multiple backup destination support - */ - public boolean isWALFileDeletable(String file) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Check if WAL file has been already backed up in backup system table " + file); - } - try (Table table = connection.getTable(tableName)) { - Get get = createGetForCheckWALFile(file); - Result res = table.get(get); - if (res.isEmpty()) { - return false; - } - return true; - } - } - - /** - * Checks if we have at least one backup session in backup system table This API is used by - * BackupLogCleaner - * @return true, if - at least one session exists in backup system table table - * @throws IOException exception - */ - public boolean hasBackupSessions() throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Has backup sessions from backup system table"); - } - boolean result = false; - Scan scan = createScanForBackupHistory(); - scan.setCaching(1); - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - if (scanner.next() != null) { - result = true; - } - return result; - } - } - - /** - * BACKUP SETS - */ - - /** - * Get backup set list - * @return backup set list - * @throws IOException - */ - public List listBackupSets() throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace(" Backup set list"); - } - List list = new ArrayList(); - Table table = null; - ResultScanner scanner = null; - try { - table = connection.getTable(tableName); - Scan scan = createScanForBackupSetList(); - scan.setMaxVersions(1); - scanner = table.getScanner(scan); - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - list.add(cellKeyToBackupSetName(res.current())); - } - return list; - } finally { - if (scanner != null) { - scanner.close(); - } - if (table != null) { - table.close(); - } - } - } - - /** - * Get backup set description (list of tables) - * @param name set's name - * @return list of tables in a backup set - * @throws IOException - */ - public List describeBackupSet(String name) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace(" Backup set describe: " + name); - } - Table table = null; - try { - table = connection.getTable(tableName); - Get get = createGetForBackupSet(name); - Result res = table.get(get); - if (res.isEmpty()) return null; - res.advance(); - String[] tables = cellValueToBackupSet(res.current()); - return toList(tables); - } finally { - if (table != null) { - table.close(); - } - } - } - - private List toList(String[] tables) { - List list = new ArrayList(tables.length); - for (String name : tables) { - list.add(TableName.valueOf(name)); - } - return list; - } - - /** - * Add backup set (list of tables) - * @param name set name - * @param newTables list of tables, comma-separated - * @throws IOException - */ - public void addToBackupSet(String name, String[] newTables) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Backup set add: " + name + " tables [" + StringUtils.join(newTables, " ") + "]"); - } - Table table = null; - String[] union = null; - try { - table = connection.getTable(tableName); - Get get = createGetForBackupSet(name); - Result res = table.get(get); - if (res.isEmpty()) { - union = newTables; - } else { - res.advance(); - String[] tables = cellValueToBackupSet(res.current()); - union = merge(tables, newTables); - } - Put put = createPutForBackupSet(name, union); - table.put(put); - } finally { - if (table != null) { - table.close(); - } - } - } - - private String[] merge(String[] tables, String[] newTables) { - List list = new ArrayList(); - // Add all from tables - for (String t : tables) { - list.add(t); - } - for (String nt : newTables) { - if (list.contains(nt)) continue; - list.add(nt); - } - String[] arr = new String[list.size()]; - list.toArray(arr); - return arr; - } - - /** - * Remove tables from backup set (list of tables) - * @param name set name - * @param toRemove list of tables - * @throws IOException - */ - public void removeFromBackupSet(String name, String[] toRemove) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace(" Backup set remove from : " + name + " tables [" + StringUtils.join(toRemove, " ") - + "]"); - } - Table table = null; - String[] disjoint = null; - String[] tables = null; - try { - table = connection.getTable(tableName); - Get get = createGetForBackupSet(name); - Result res = table.get(get); - if (res.isEmpty()) { - LOG.warn("Backup set '" + name + "' not found."); - return; - } else { - res.advance(); - tables = cellValueToBackupSet(res.current()); - disjoint = disjoin(tables, toRemove); - } - if (disjoint.length > 0 && disjoint.length != tables.length) { - Put put = createPutForBackupSet(name, disjoint); - table.put(put); - } else if (disjoint.length == tables.length) { - LOG.warn("Backup set '" + name + "' does not contain tables [" - + StringUtils.join(toRemove, " ") + "]"); - } else { // disjoint.length == 0 and tables.length >0 - // Delete backup set - LOG.info("Backup set '" + name + "' is empty. Deleting."); - deleteBackupSet(name); - } - } finally { - if (table != null) { - table.close(); - } - } - } - - private String[] disjoin(String[] tables, String[] toRemove) { - List list = new ArrayList(); - // Add all from tables - for (String t : tables) { - list.add(t); - } - for (String nt : toRemove) { - if (list.contains(nt)) { - list.remove(nt); - } - } - String[] arr = new String[list.size()]; - list.toArray(arr); - return arr; - } - - /** - * Delete backup set - * @param name set's name - * @throws IOException - */ - public void deleteBackupSet(String name) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace(" Backup set delete: " + name); - } - Table table = null; - try { - table = connection.getTable(tableName); - Delete del = createDeleteForBackupSet(name); - table.delete(del); - } finally { - if (table != null) { - table.close(); - } - } - } - - /** - * Get backup system table descriptor - * @return table's descriptor - */ - public static HTableDescriptor getSystemTableDescriptor(Configuration conf) { - - HTableDescriptor tableDesc = new HTableDescriptor(getTableName(conf)); - HColumnDescriptor colSessionsDesc = new HColumnDescriptor(SESSIONS_FAMILY); - colSessionsDesc.setMaxVersions(1); - // Time to keep backup sessions (secs) - Configuration config = HBaseConfiguration.create(); - int ttl = - config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY, - BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT); - colSessionsDesc.setTimeToLive(ttl); - tableDesc.addFamily(colSessionsDesc); - HColumnDescriptor colMetaDesc = new HColumnDescriptor(META_FAMILY); - tableDesc.addFamily(colMetaDesc); - return tableDesc; - } - - public static TableName getTableName(Configuration conf) { - String name = - conf.get(BackupRestoreConstants.BACKUP_SYSTEM_TABLE_NAME_KEY, - BackupRestoreConstants.BACKUP_SYSTEM_TABLE_NAME_DEFAULT); - return TableName.valueOf(name); - } - - public static String getTableNameAsString(Configuration conf) { - return getTableName(conf).getNameAsString(); - } - - public static String getSnapshotName(Configuration conf) { - return "snapshot_" + getTableNameAsString(conf).replace(":", "_"); - } - - /** - * Creates Put operation for a given backup info object - * @param context backup info - * @return put operation - * @throws IOException exception - */ - private Put createPutForBackupInfo(BackupInfo context) throws IOException { - Put put = new Put(rowkey(BACKUP_INFO_PREFIX, context.getBackupId())); - put.addColumn(BackupSystemTable.SESSIONS_FAMILY, Bytes.toBytes("context"), - context.toByteArray()); - return put; - } - - /** - * Creates Get operation for a given backup id - * @param backupId backup's ID - * @return get operation - * @throws IOException exception - */ - private Get createGetForBackupInfo(String backupId) throws IOException { - Get get = new Get(rowkey(BACKUP_INFO_PREFIX, backupId)); - get.addFamily(BackupSystemTable.SESSIONS_FAMILY); - get.setMaxVersions(1); - return get; - } - - /** - * Creates Delete operation for a given backup id - * @param backupId backup's ID - * @return delete operation - * @throws IOException exception - */ - private Delete createDeleteForBackupInfo(String backupId) { - Delete del = new Delete(rowkey(BACKUP_INFO_PREFIX, backupId)); - del.addFamily(BackupSystemTable.SESSIONS_FAMILY); - return del; - } - - /** - * Converts Result to BackupInfo - * @param res HBase result - * @return backup info instance - * @throws IOException exception - */ - private BackupInfo resultToBackupInfo(Result res) throws IOException { - res.advance(); - Cell cell = res.current(); - return cellToBackupInfo(cell); - } - - /** - * Creates Get operation to retrieve start code from backup system table - * @return get operation - * @throws IOException exception - */ - private Get createGetForStartCode(String rootPath) throws IOException { - Get get = new Get(rowkey(START_CODE_ROW, rootPath)); - get.addFamily(BackupSystemTable.META_FAMILY); - get.setMaxVersions(1); - return get; - } - - /** - * Creates Put operation to store start code to backup system table - * @return put operation - * @throws IOException exception - */ - private Put createPutForStartCode(String startCode, String rootPath) { - Put put = new Put(rowkey(START_CODE_ROW, rootPath)); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("startcode"), - Bytes.toBytes(startCode)); - return put; - } - - /** - * Creates Get to retrieve incremental backup table set from backup system table - * @return get operation - * @throws IOException exception - */ - private Get createGetForIncrBackupTableSet(String backupRoot) throws IOException { - Get get = new Get(rowkey(INCR_BACKUP_SET, backupRoot)); - get.addFamily(BackupSystemTable.META_FAMILY); - get.setMaxVersions(1); - return get; - } - - /** - * Creates Put to store incremental backup table set - * @param tables tables - * @return put operation - */ - private Put createPutForIncrBackupTableSet(Set tables, String backupRoot) { - Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot)); - for (TableName table : tables) { - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()), - EMPTY_VALUE); - } - return put; - } - - /** - * Creates Delete for incremental backup table set - * @param backupRoot backup root - * @return delete operation - */ - private Delete createDeleteForIncrBackupTableSet(String backupRoot) { - Delete delete = new Delete(rowkey(INCR_BACKUP_SET, backupRoot)); - delete.addFamily(BackupSystemTable.META_FAMILY); - return delete; - } - - /** - * Creates Scan operation to load backup history - * @return scan operation - */ - private Scan createScanForBackupHistory() { - Scan scan = new Scan(); - byte[] startRow = Bytes.toBytes(BACKUP_INFO_PREFIX); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.SESSIONS_FAMILY); - scan.setMaxVersions(1); - return scan; - } - - /** - * Converts cell to backup info instance. - * @param current current cell - * @return backup backup info instance - * @throws IOException exception - */ - private BackupInfo cellToBackupInfo(Cell current) throws IOException { - byte[] data = CellUtil.cloneValue(current); - return BackupInfo.fromByteArray(data); - } - - /** - * Creates Put to write RS last roll log timestamp map - * @param table table - * @param smap map, containing RS:ts - * @return put operation - */ - private Put createPutForWriteRegionServerLogTimestamp(TableName table, byte[] smap, - String backupRoot) { - Put put = new Put(rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot, NULL, table.getNameAsString())); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("log-roll-map"), smap); - return put; - } - - /** - * Creates Scan to load table-> { RS -> ts} map of maps - * @return scan operation - */ - private Scan createScanForReadLogTimestampMap(String backupRoot) { - Scan scan = new Scan(); - byte[] startRow = rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - - return scan; - } - - /** - * Get table name from rowkey - * @param cloneRow rowkey - * @return table name - */ - private String getTableNameForReadLogTimestampMap(byte[] cloneRow) { - String s = Bytes.toString(cloneRow); - int index = s.lastIndexOf(NULL); - return s.substring(index + 1); - } - - /** - * Creates Put to store RS last log result - * @param server server name - * @param timestamp log roll result (timestamp) - * @return put operation - */ - private Put createPutForRegionServerLastLogRollResult(String server, Long timestamp, - String backupRoot) { - Put put = new Put(rowkey(RS_LOG_TS_PREFIX, backupRoot, NULL, server)); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("rs-log-ts"), - Bytes.toBytes(timestamp)); - return put; - } - - /** - * Creates Scan operation to load last RS log roll results - * @return scan operation - */ - private Scan createScanForReadRegionServerLastLogRollResult(String backupRoot) { - Scan scan = new Scan(); - byte[] startRow = rowkey(RS_LOG_TS_PREFIX, backupRoot); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - scan.setMaxVersions(1); - - return scan; - } - - /** - * Get server's name from rowkey - * @param row rowkey - * @return server's name - */ - private String getServerNameForReadRegionServerLastLogRollResult(byte[] row) { - String s = Bytes.toString(row); - int index = s.lastIndexOf(NULL); - return s.substring(index + 1); - } - - /* - * Creates Put's for bulk load resulting from running LoadIncrementalHFiles - */ - static List createPutForCommittedBulkload(TableName table, byte[] region, - Map> finalPaths) { - List puts = new ArrayList<>(); - for (Map.Entry> entry : finalPaths.entrySet()) { - for (Path path : entry.getValue()) { - String file = path.toString(); - int lastSlash = file.lastIndexOf("/"); - String filename = file.substring(lastSlash + 1); - Put put = - new Put(rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM, - Bytes.toString(region), BLK_LD_DELIM, filename)); - put.addColumn(BackupSystemTable.META_FAMILY, TBL_COL, table.getName()); - put.addColumn(BackupSystemTable.META_FAMILY, FAM_COL, entry.getKey()); - put.addColumn(BackupSystemTable.META_FAMILY, PATH_COL, file.getBytes()); - put.addColumn(BackupSystemTable.META_FAMILY, STATE_COL, BL_COMMIT); - puts.add(put); - LOG.debug("writing done bulk path " + file + " for " + table + " " + Bytes.toString(region)); - } - } - return puts; - } - - public static void snapshot(Connection conn) throws IOException { - - try (Admin admin = conn.getAdmin();) { - Configuration conf = conn.getConfiguration(); - admin.snapshot(BackupSystemTable.getSnapshotName(conf), BackupSystemTable.getTableName(conf)); - } - } - - public static void restoreFromSnapshot(Connection conn) throws IOException { - - Configuration conf = conn.getConfiguration(); - LOG.debug("Restoring " + BackupSystemTable.getTableNameAsString(conf) + " from snapshot"); - try (Admin admin = conn.getAdmin();) { - String snapshotName = BackupSystemTable.getSnapshotName(conf); - if (snapshotExists(admin, snapshotName)) { - admin.disableTable(BackupSystemTable.getTableName(conf)); - admin.restoreSnapshot(snapshotName); - admin.enableTable(BackupSystemTable.getTableName(conf)); - LOG.debug("Done restoring backup system table"); - } else { - // Snapshot does not exists, i.e completeBackup failed after - // deleting backup system table snapshot - // In this case we log WARN and proceed - LOG.warn("Could not restore backup system table. Snapshot " + snapshotName - + " does not exists."); - } - } - } - - protected static boolean snapshotExists(Admin admin, String snapshotName) throws IOException { - - List list = admin.listSnapshots(); - for (SnapshotDescription desc : list) { - if (desc.getName().equals(snapshotName)) { - return true; - } - } - return false; - } - - public static boolean snapshotExists(Connection conn) throws IOException { - return snapshotExists(conn.getAdmin(), getSnapshotName(conn.getConfiguration())); - } - - public static void deleteSnapshot(Connection conn) throws IOException { - - Configuration conf = conn.getConfiguration(); - LOG.debug("Deleting " + BackupSystemTable.getSnapshotName(conf) + " from the system"); - try (Admin admin = conn.getAdmin();) { - String snapshotName = BackupSystemTable.getSnapshotName(conf); - if (snapshotExists(admin, snapshotName)) { - admin.deleteSnapshot(snapshotName); - LOG.debug("Done deleting backup system table snapshot"); - } else { - LOG.error("Snapshot " + snapshotName + " does not exists"); - } - } - } - - /* - * Creates Put's for bulk load resulting from running LoadIncrementalHFiles - */ - static List createPutForPreparedBulkload(TableName table, byte[] region, - final byte[] family, final List> pairs) { - List puts = new ArrayList<>(); - for (Pair pair : pairs) { - Path path = pair.getSecond(); - String file = path.toString(); - int lastSlash = file.lastIndexOf("/"); - String filename = file.substring(lastSlash + 1); - Put put = - new Put(rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM, Bytes.toString(region), - BLK_LD_DELIM, filename)); - put.addColumn(BackupSystemTable.META_FAMILY, TBL_COL, table.getName()); - put.addColumn(BackupSystemTable.META_FAMILY, FAM_COL, family); - put.addColumn(BackupSystemTable.META_FAMILY, PATH_COL, file.getBytes()); - put.addColumn(BackupSystemTable.META_FAMILY, STATE_COL, BL_PREPARE); - puts.add(put); - LOG.debug("writing raw bulk path " + file + " for " + table + " " + Bytes.toString(region)); - } - return puts; - } - - public static List createDeleteForOrigBulkLoad(List lst) { - List lstDels = new ArrayList<>(); - for (TableName table : lst) { - Delete del = new Delete(rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM)); - del.addFamily(BackupSystemTable.META_FAMILY); - lstDels.add(del); - } - return lstDels; - } - - private Put createPutForDeleteOperation(String[] backupIdList) { - - byte[] value = Bytes.toBytes(StringUtils.join(backupIdList, ",")); - Put put = new Put(DELETE_OP_ROW); - put.addColumn(META_FAMILY, FAM_COL, value); - return put; - } - - private Delete createDeleteForBackupDeleteOperation() { - - Delete delete = new Delete(DELETE_OP_ROW); - delete.addFamily(META_FAMILY); - return delete; - } - - private Get createGetForDeleteOperation() { - - Get get = new Get(DELETE_OP_ROW); - get.addFamily(META_FAMILY); - return get; - } - - public void startDeleteOperation(String[] backupIdList) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Start delete operation for backups: " + StringUtils.join(backupIdList)); - } - Put put = createPutForDeleteOperation(backupIdList); - try (Table table = connection.getTable(tableName)) { - table.put(put); - } - } - - public void finishDeleteOperation() throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Finsih delete operation for backup ids "); - } - Delete delete = createDeleteForBackupDeleteOperation(); - try (Table table = connection.getTable(tableName)) { - table.delete(delete); - } - } - - public String[] getListOfBackupIdsFromDeleteOperation() throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Get delete operation for backup ids "); - } - Get get = createGetForDeleteOperation(); - try (Table table = connection.getTable(tableName)) { - Result res = table.get(get); - if (res.isEmpty()) { - return null; - } - Cell cell = res.listCells().get(0); - byte[] val = CellUtil.cloneValue(cell); - if (val.length == 0) { - return null; - } - return new String(val).split(","); - } - } - - private Put createPutForMergeOperation(String[] backupIdList) { - - byte[] value = Bytes.toBytes(StringUtils.join(backupIdList, ",")); - Put put = new Put(MERGE_OP_ROW); - put.addColumn(META_FAMILY, FAM_COL, value); - return put; - } - - public boolean isMergeInProgress() throws IOException { - Get get = new Get(MERGE_OP_ROW); - try (Table table = connection.getTable(tableName)) { - Result res = table.get(get); - if (res.isEmpty()) { - return false; - } - return true; - } - } - - private Put createPutForUpdateTablesForMerge(List tables) { - - byte[] value = Bytes.toBytes(StringUtils.join(tables, ",")); - Put put = new Put(MERGE_OP_ROW); - put.addColumn(META_FAMILY, PATH_COL, value); - return put; - } - - private Delete createDeleteForBackupMergeOperation() { - - Delete delete = new Delete(MERGE_OP_ROW); - delete.addFamily(META_FAMILY); - return delete; - } - - private Get createGetForMergeOperation() { - - Get get = new Get(MERGE_OP_ROW); - get.addFamily(META_FAMILY); - return get; - } - - public void startMergeOperation(String[] backupIdList) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Start merge operation for backups: " + StringUtils.join(backupIdList)); - } - Put put = createPutForMergeOperation(backupIdList); - try (Table table = connection.getTable(tableName)) { - table.put(put); - } - } - - public void updateProcessedTablesForMerge(List tables) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Update tables for merge : " + StringUtils.join(tables, ",")); - } - Put put = createPutForUpdateTablesForMerge(tables); - try (Table table = connection.getTable(tableName)) { - table.put(put); - } - } - - public void finishMergeOperation() throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Finsih merge operation for backup ids "); - } - Delete delete = createDeleteForBackupMergeOperation(); - try (Table table = connection.getTable(tableName)) { - table.delete(delete); - } - } - - public String[] getListOfBackupIdsFromMergeOperation() throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("Get backup ids for merge operation"); - } - Get get = createGetForMergeOperation(); - try (Table table = connection.getTable(tableName)) { - Result res = table.get(get); - if (res.isEmpty()) { - return null; - } - Cell cell = res.listCells().get(0); - byte[] val = CellUtil.cloneValue(cell); - if (val.length == 0) { - return null; - } - return new String(val).split(","); - } - } - - static Scan createScanForOrigBulkLoadedFiles(TableName table) throws IOException { - Scan scan = new Scan(); - byte[] startRow = rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.withStartRow(startRow); - scan.withStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - scan.setMaxVersions(1); - return scan; - } - - static String getTableNameFromOrigBulkLoadRow(String rowStr) { - String[] parts = rowStr.split(BLK_LD_DELIM); - return parts[1]; - } - - static String getRegionNameFromOrigBulkLoadRow(String rowStr) { - // format is bulk : namespace : table : region : file - String[] parts = rowStr.split(BLK_LD_DELIM); - int idx = 3; - if (parts.length == 4) { - // the table is in default namespace - idx = 2; - } - LOG.debug("bulk row string " + rowStr + " region " + parts[idx]); - return parts[idx]; - } - - /* - * Used to query bulk loaded hfiles which have been copied by incremental backup - * @param backupId the backup Id. It can be null when querying for all tables - * @return the Scan object - */ - static Scan createScanForBulkLoadedFiles(String backupId) throws IOException { - Scan scan = new Scan(); - byte[] startRow = - backupId == null ? BULK_LOAD_PREFIX_BYTES : rowkey(BULK_LOAD_PREFIX, backupId - + BLK_LD_DELIM); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - // scan.setTimeRange(lower, Long.MAX_VALUE); - scan.addFamily(BackupSystemTable.META_FAMILY); - scan.setMaxVersions(1); - return scan; - } - - static Put createPutForBulkLoadedFile(TableName tn, byte[] fam, String p, String backupId, - long ts, int idx) { - Put put = new Put(rowkey(BULK_LOAD_PREFIX, backupId + BLK_LD_DELIM + ts + BLK_LD_DELIM + idx)); - put.addColumn(BackupSystemTable.META_FAMILY, TBL_COL, tn.getName()); - put.addColumn(BackupSystemTable.META_FAMILY, FAM_COL, fam); - put.addColumn(BackupSystemTable.META_FAMILY, PATH_COL, p.getBytes()); - return put; - } - - /** - * Creates put list for list of WAL files - * @param files list of WAL file paths - * @param backupId backup id - * @return put list - * @throws IOException exception - */ - private List - createPutsForAddWALFiles(List files, String backupId, String backupRoot) - throws IOException { - - List puts = new ArrayList(); - for (String file : files) { - Put put = new Put(rowkey(WALS_PREFIX, BackupUtils.getUniqueWALFileNamePart(file))); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("backupId"), - Bytes.toBytes(backupId)); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("file"), Bytes.toBytes(file)); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("root"), Bytes.toBytes(backupRoot)); - puts.add(put); - } - return puts; - } - - /** - * Creates Scan operation to load WALs - * @param backupRoot path to backup destination - * @return scan operation - */ - private Scan createScanForGetWALs(String backupRoot) { - // TODO: support for backupRoot - Scan scan = new Scan(); - byte[] startRow = Bytes.toBytes(WALS_PREFIX); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - return scan; - } - - /** - * Creates Get operation for a given wal file name TODO: support for backup destination - * @param file file - * @return get operation - * @throws IOException exception - */ - private Get createGetForCheckWALFile(String file) throws IOException { - Get get = new Get(rowkey(WALS_PREFIX, BackupUtils.getUniqueWALFileNamePart(file))); - // add backup root column - get.addFamily(BackupSystemTable.META_FAMILY); - return get; - } - - /** - * Creates Scan operation to load backup set list - * @return scan operation - */ - private Scan createScanForBackupSetList() { - Scan scan = new Scan(); - byte[] startRow = Bytes.toBytes(SET_KEY_PREFIX); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - return scan; - } - - /** - * Creates Get operation to load backup set content - * @return get operation - */ - private Get createGetForBackupSet(String name) { - Get get = new Get(rowkey(SET_KEY_PREFIX, name)); - get.addFamily(BackupSystemTable.META_FAMILY); - return get; - } - - /** - * Creates Delete operation to delete backup set content - * @param name backup set's name - * @return delete operation - */ - private Delete createDeleteForBackupSet(String name) { - Delete del = new Delete(rowkey(SET_KEY_PREFIX, name)); - del.addFamily(BackupSystemTable.META_FAMILY); - return del; - } - - /** - * Creates Put operation to update backup set content - * @param name backup set's name - * @param tables list of tables - * @return put operation - */ - private Put createPutForBackupSet(String name, String[] tables) { - Put put = new Put(rowkey(SET_KEY_PREFIX, name)); - byte[] value = convertToByteArray(tables); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("tables"), value); - return put; - } - - private byte[] convertToByteArray(String[] tables) { - return StringUtils.join(tables, ",").getBytes(); - } - - /** - * Converts cell to backup set list. - * @param current current cell - * @return backup set as array of table names - * @throws IOException - */ - private String[] cellValueToBackupSet(Cell current) throws IOException { - byte[] data = CellUtil.cloneValue(current); - if (data != null && data.length > 0) { - return Bytes.toString(data).split(","); - } else { - return new String[0]; - } - } - - /** - * Converts cell key to backup set name. - * @param current current cell - * @return backup set name - * @throws IOException - */ - private String cellKeyToBackupSetName(Cell current) throws IOException { - byte[] data = CellUtil.cloneRow(current); - return Bytes.toString(data).substring(SET_KEY_PREFIX.length()); - } - - private static byte[] rowkey(String s, String... other) { - StringBuilder sb = new StringBuilder(s); - for (String ss : other) { - sb.append(ss); - } - return sb.toString().getBytes(); - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java deleted file mode 100644 index cb8e092e4a1..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_ATTEMPTS_PAUSE_MS_KEY; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_MAX_ATTEMPTS_KEY; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_MAX_ATTEMPTS; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupCopyJob; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -/** - * Full table backup implementation - * - */ -@InterfaceAudience.Private -public class FullTableBackupClient extends TableBackupClient { - private static final Log LOG = LogFactory.getLog(FullTableBackupClient.class); - - public FullTableBackupClient() { - } - - public FullTableBackupClient(final Connection conn, final String backupId, BackupRequest request) - throws IOException { - super(conn, backupId, request); - } - - /** - * Do snapshot copy. - * @param backupInfo backup info - * @throws Exception exception - */ - protected void snapshotCopy(BackupInfo backupInfo) throws Exception { - LOG.info("Snapshot copy is starting."); - - // set overall backup phase: snapshot_copy - backupInfo.setPhase(BackupPhase.SNAPSHOTCOPY); - - // call ExportSnapshot to copy files based on hbase snapshot for backup - // ExportSnapshot only support single snapshot export, need loop for multiple tables case - BackupCopyJob copyService = BackupRestoreFactory.getBackupCopyJob(conf); - - // number of snapshots matches number of tables - float numOfSnapshots = backupInfo.getSnapshotNames().size(); - - LOG.debug("There are " + (int) numOfSnapshots + " snapshots to be copied."); - - for (TableName table : backupInfo.getTables()) { - // Currently we simply set the sub copy tasks by counting the table snapshot number, we can - // calculate the real files' size for the percentage in the future. - // backupCopier.setSubTaskPercntgInWholeTask(1f / numOfSnapshots); - int res = 0; - String[] args = new String[4]; - args[0] = "-snapshot"; - args[1] = backupInfo.getSnapshotName(table); - args[2] = "-copy-to"; - args[3] = backupInfo.getTableBackupDir(table); - - String jobname = "Full-Backup_" + backupInfo.getBackupId() + "_" + table.getNameAsString(); - if (LOG.isDebugEnabled()) { - LOG.debug("Setting snapshot copy job name to : " + jobname); - } - conf.set(JOB_NAME_CONF_KEY, jobname); - - LOG.debug("Copy snapshot " + args[1] + " to " + args[3]); - res = copyService.copy(backupInfo, backupManager, conf, BackupType.FULL, args); - - // if one snapshot export failed, do not continue for remained snapshots - if (res != 0) { - LOG.error("Exporting Snapshot " + args[1] + " failed with return code: " + res + "."); - - throw new IOException("Failed of exporting snapshot " + args[1] + " to " + args[3] - + " with reason code " + res); - } - - conf.unset(JOB_NAME_CONF_KEY); - LOG.info("Snapshot copy " + args[1] + " finished."); - } - } - - /** - * Backup request execution - * @throws IOException - */ - @Override - public void execute() throws IOException { - try (Admin admin = conn.getAdmin();) { - - // Begin BACKUP - beginBackup(backupManager, backupInfo); - String savedStartCode = null; - boolean firstBackup = false; - // do snapshot for full table backup - - savedStartCode = backupManager.readBackupStartCode(); - firstBackup = savedStartCode == null || Long.parseLong(savedStartCode) == 0L; - if (firstBackup) { - // This is our first backup. Let's put some marker to system table so that we can hold the logs - // while we do the backup. - backupManager.writeBackupStartCode(0L); - } - // We roll log here before we do the snapshot. It is possible there is duplicate data - // in the log that is already in the snapshot. But if we do it after the snapshot, we - // could have data loss. - // A better approach is to do the roll log on each RS in the same global procedure as - // the snapshot. - LOG.info("Execute roll log procedure for full backup ..."); - - Map props = new HashMap(); - props.put("backupRoot", backupInfo.getBackupRootDir()); - admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, - LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); - - newTimestamps = backupManager.readRegionServerLastLogRollResult(); - if (firstBackup) { - // Updates registered log files - // We record ALL old WAL files as registered, because - // this is a first full backup in the system and these - // files are not needed for next incremental backup - List logFiles = BackupUtils.getWALFilesOlderThan(conf, newTimestamps); - backupManager.recordWALFiles(logFiles); - } - - // SNAPSHOT_TABLES: - backupInfo.setPhase(BackupPhase.SNAPSHOT); - for (TableName tableName : tableList) { - String snapshotName = - "snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime()) + "_" - + tableName.getNamespaceAsString() + "_" + tableName.getQualifierAsString(); - - snapshotTable(admin, tableName, snapshotName); - backupInfo.setSnapshotName(tableName, snapshotName); - } - - // SNAPSHOT_COPY: - // do snapshot copy - LOG.debug("snapshot copy for " + backupId); - snapshotCopy(backupInfo); - // Updates incremental backup table set - backupManager.addIncrementalBackupTableSet(backupInfo.getTables()); - - // BACKUP_COMPLETE: - // set overall backup status: complete. Here we make sure to complete the backup. - // After this checkpoint, even if entering cancel process, will let the backup finished - backupInfo.setState(BackupState.COMPLETE); - // The table list in backupInfo is good for both full backup and incremental backup. - // For incremental backup, it contains the incremental backup table set. - backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); - - HashMap> newTableSetTimestampMap = - backupManager.readLogTimestampMap(); - - Long newStartCode = - BackupUtils.getMinValue(BackupUtils - .getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); - - // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.FULL, conf); - } catch (Exception e) { - failBackup(conn, backupInfo, backupManager, e, "Unexpected BackupException : ", - BackupType.FULL, conf); - throw new IOException(e); - } - - } - - - protected void snapshotTable(Admin admin, TableName tableName, String snapshotName) - throws IOException { - - int maxAttempts = - conf.getInt(BACKUP_MAX_ATTEMPTS_KEY, DEFAULT_BACKUP_MAX_ATTEMPTS); - int pause = - conf.getInt(BACKUP_ATTEMPTS_PAUSE_MS_KEY, DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS); - int attempts = 0; - - while (attempts++ < maxAttempts) { - try { - admin.snapshot(snapshotName, tableName); - return; - } catch (IOException ee) { - LOG.warn("Snapshot attempt " + attempts + " failed for table " + tableName - + ", sleeping for " + pause + "ms", ee); - if (attempts < maxAttempts) { - try { - Thread.sleep(pause); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - } - } - throw new IOException("Failed to snapshot table "+ tableName); - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java deleted file mode 100644 index d9258c5729e..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ /dev/null @@ -1,387 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable.WALItem; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; - -/** - * After a full backup was created, the incremental backup will only store the changes made after - * the last full or incremental backup. Creating the backup copies the logfiles in .logs and - * .oldlogs since the last backup timestamp. - */ -@InterfaceAudience.Private -public class IncrementalBackupManager extends BackupManager { - public static final Log LOG = LogFactory.getLog(IncrementalBackupManager.class); - - public IncrementalBackupManager(Connection conn, Configuration conf) throws IOException { - super(conn, conf); - } - - /** - * Obtain the list of logs that need to be copied out for this incremental backup. The list is set - * in BackupInfo. - * @return The new HashMap of RS log time stamps after the log roll for this incremental backup. - * @throws IOException exception - */ - public HashMap getIncrBackupLogFileMap() - throws IOException { - List logList; - HashMap newTimestamps; - HashMap previousTimestampMins; - - String savedStartCode = readBackupStartCode(); - - // key: tableName - // value: - HashMap> previousTimestampMap = readLogTimestampMap(); - - previousTimestampMins = BackupUtils.getRSLogTimestampMins(previousTimestampMap); - - if (LOG.isDebugEnabled()) { - LOG.debug("StartCode " + savedStartCode + "for backupID " + backupInfo.getBackupId()); - } - // get all new log files from .logs and .oldlogs after last TS and before new timestamp - if (savedStartCode == null || previousTimestampMins == null - || previousTimestampMins.isEmpty()) { - throw new IOException( - "Cannot read any previous back up timestamps from backup system table. " - + "In order to create an incremental backup, at least one full backup is needed."); - } - - LOG.info("Execute roll log procedure for incremental backup ..."); - HashMap props = new HashMap(); - props.put("backupRoot", backupInfo.getBackupRootDir()); - - try (Admin admin = conn.getAdmin();) { - - admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, - LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); - - } - newTimestamps = readRegionServerLastLogRollResult(); - - logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps, conf, savedStartCode); - List logFromSystemTable = - getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo() - .getBackupRootDir()); - logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable); - backupInfo.setIncrBackupFileList(logList); - - return newTimestamps; - } - - /** - * Get list of WAL files eligible for incremental backup - * @return list of WAL files - * @throws IOException - */ - public List getIncrBackupLogFileList() - throws IOException { - List logList; - HashMap newTimestamps; - HashMap previousTimestampMins; - - String savedStartCode = readBackupStartCode(); - - // key: tableName - // value: - HashMap> previousTimestampMap = readLogTimestampMap(); - - previousTimestampMins = BackupUtils.getRSLogTimestampMins(previousTimestampMap); - - if (LOG.isDebugEnabled()) { - LOG.debug("StartCode " + savedStartCode + "for backupID " + backupInfo.getBackupId()); - } - // get all new log files from .logs and .oldlogs after last TS and before new timestamp - if (savedStartCode == null || previousTimestampMins == null - || previousTimestampMins.isEmpty()) { - throw new IOException( - "Cannot read any previous back up timestamps from backup system table. " - + "In order to create an incremental backup, at least one full backup is needed."); - } - - newTimestamps = readRegionServerLastLogRollResult(); - - logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps, conf, savedStartCode); - List logFromSystemTable = - getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo() - .getBackupRootDir()); - - logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable); - backupInfo.setIncrBackupFileList(logList); - - return logList; - } - - - private List excludeAlreadyBackedUpWALs(List logList, - List logFromSystemTable) { - - Set walFileNameSet = convertToSet(logFromSystemTable); - - List list = new ArrayList(); - for (int i=0; i < logList.size(); i++) { - Path p = new Path(logList.get(i)); - String name = p.getName(); - if (walFileNameSet.contains(name)) continue; - list.add(logList.get(i)); - } - return list; - } - - /** - * Create Set of WAL file names (not full path names) - * @param logFromSystemTable - * @return set of WAL file names - */ - private Set convertToSet(List logFromSystemTable) { - - Set set = new HashSet(); - for (int i=0; i < logFromSystemTable.size(); i++) { - WALItem item = logFromSystemTable.get(i); - set.add(item.walFile); - } - return set; - } - - /** - * For each region server: get all log files newer than the last timestamps, but not newer than - * the newest timestamps. - * @param olderTimestamps timestamp map for each region server of the last backup. - * @param newestTimestamps timestamp map for each region server that the backup should lead to. - * @return list of log files which needs to be added to this backup - * @throws IOException - */ - private List getLogFilesFromBackupSystem(HashMap olderTimestamps, - HashMap newestTimestamps, String backupRoot) throws IOException { - List logFiles = new ArrayList(); - Iterator it = getWALFilesFromBackupSystem(); - while (it.hasNext()) { - WALItem item = it.next(); - String rootDir = item.getBackupRoot(); - if (!rootDir.equals(backupRoot)) { - continue; - } - String walFileName = item.getWalFile(); - String server = BackupUtils.parseHostNameFromLogFile(new Path(walFileName)); - if (server == null) { - continue; - } - Long tss = getTimestamp(walFileName); - Long oldTss = olderTimestamps.get(server); - Long newTss = newestTimestamps.get(server); - if (oldTss == null) { - logFiles.add(item); - continue; - } - if (newTss == null) { - newTss = Long.MAX_VALUE; - } - if (tss > oldTss && tss < newTss) { - logFiles.add(item); - } - } - return logFiles; - } - - private Long getTimestamp(String walFileName) { - int index = walFileName.lastIndexOf(BackupUtils.LOGNAME_SEPARATOR); - return Long.parseLong(walFileName.substring(index + 1)); - } - - /** - * For each region server: get all log files newer than the last timestamps but not newer than the - * newest timestamps. - * @param olderTimestamps the timestamp for each region server of the last backup. - * @param newestTimestamps the timestamp for each region server that the backup should lead to. - * @param conf the Hadoop and Hbase configuration - * @param savedStartCode the startcode (timestamp) of last successful backup. - * @return a list of log files to be backed up - * @throws IOException exception - */ - private List getLogFilesForNewBackup(HashMap olderTimestamps, - HashMap newestTimestamps, Configuration conf, String savedStartCode) - throws IOException { - LOG.debug("In getLogFilesForNewBackup()\n" + "olderTimestamps: " + olderTimestamps - + "\n newestTimestamps: " + newestTimestamps); - Path rootdir = FSUtils.getRootDir(conf); - Path logDir = new Path(rootdir, HConstants.HREGION_LOGDIR_NAME); - Path oldLogDir = new Path(rootdir, HConstants.HREGION_OLDLOGDIR_NAME); - FileSystem fs = rootdir.getFileSystem(conf); - NewestLogFilter pathFilter = new NewestLogFilter(); - - List resultLogFiles = new ArrayList(); - List newestLogs = new ArrayList(); - - /* - * The old region servers and timestamps info we kept in backup system table may be out of sync - * if new region server is added or existing one lost. We'll deal with it here when processing - * the logs. If data in backup system table has more hosts, just ignore it. If the .logs - * directory includes more hosts, the additional hosts will not have old timestamps to compare - * with. We'll just use all the logs in that directory. We always write up-to-date region server - * and timestamp info to backup system table at the end of successful backup. - */ - - FileStatus[] rss; - Path p; - String host; - Long oldTimeStamp; - String currentLogFile; - long currentLogTS; - - // Get the files in .logs. - rss = fs.listStatus(logDir); - for (FileStatus rs : rss) { - p = rs.getPath(); - host = BackupUtils.parseHostNameFromLogFile(p); - if (host == null) { - continue; - } - FileStatus[] logs; - oldTimeStamp = olderTimestamps.get(host); - // It is possible that there is no old timestamp in backup system table for this host if - // this region server is newly added after our last backup. - if (oldTimeStamp == null) { - logs = fs.listStatus(p); - } else { - pathFilter.setLastBackupTS(oldTimeStamp); - logs = fs.listStatus(p, pathFilter); - } - for (FileStatus log : logs) { - LOG.debug("currentLogFile: " + log.getPath().toString()); - if (AbstractFSWALProvider.isMetaFile(log.getPath())) { - if (LOG.isDebugEnabled()) { - LOG.debug("Skip hbase:meta log file: " + log.getPath().getName()); - } - continue; - } - currentLogFile = log.getPath().toString(); - resultLogFiles.add(currentLogFile); - currentLogTS = BackupUtils.getCreationTime(log.getPath()); - // newestTimestamps is up-to-date with the current list of hosts - // so newestTimestamps.get(host) will not be null. - if (currentLogTS > newestTimestamps.get(host)) { - newestLogs.add(currentLogFile); - } - } - } - - // Include the .oldlogs files too. - FileStatus[] oldlogs = fs.listStatus(oldLogDir); - for (FileStatus oldlog : oldlogs) { - p = oldlog.getPath(); - currentLogFile = p.toString(); - if (AbstractFSWALProvider.isMetaFile(p)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Skip .meta log file: " + currentLogFile); - } - continue; - } - host = BackupUtils.parseHostFromOldLog(p); - if (host == null) { - continue; - } - currentLogTS = BackupUtils.getCreationTime(p); - oldTimeStamp = olderTimestamps.get(host); - /* - * It is possible that there is no old timestamp in backup system table for this host. At the - * time of our last backup operation, this rs did not exist. The reason can be one of the two: - * 1. The rs already left/crashed. Its logs were moved to .oldlogs. 2. The rs was added after - * our last backup. - */ - if (oldTimeStamp == null) { - if (currentLogTS < Long.parseLong(savedStartCode)) { - // This log file is really old, its region server was before our last backup. - continue; - } else { - resultLogFiles.add(currentLogFile); - } - } else if (currentLogTS > oldTimeStamp) { - resultLogFiles.add(currentLogFile); - } - - // It is possible that a host in .oldlogs is an obsolete region server - // so newestTimestamps.get(host) here can be null. - // Even if these logs belong to a obsolete region server, we still need - // to include they to avoid loss of edits for backup. - Long newTimestamp = newestTimestamps.get(host); - if (newTimestamp != null && currentLogTS > newTimestamp) { - newestLogs.add(currentLogFile); - } - } - // remove newest log per host because they are still in use - resultLogFiles.removeAll(newestLogs); - return resultLogFiles; - } - - static class NewestLogFilter implements PathFilter { - private Long lastBackupTS = 0L; - - public NewestLogFilter() { - } - - protected void setLastBackupTS(Long ts) { - this.lastBackupTS = ts; - } - - @Override - public boolean accept(Path path) { - // skip meta table log -- ts.meta file - if (AbstractFSWALProvider.isMetaFile(path)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Skip .meta log file: " + path.getName()); - } - return false; - } - long timestamp; - try { - timestamp = BackupUtils.getCreationTime(path); - return timestamp > lastBackupTS; - } catch (Exception e) { - LOG.warn("Cannot read timestamp of log file " + path); - return false; - } - } - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java deleted file mode 100644 index 00baeb70a7e..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ /dev/null @@ -1,404 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupCopyJob; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyJob; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.mapreduce.WALPlayer; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HFileArchiveUtil; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.apache.hadoop.util.Tool; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Incremental backup implementation. - * See the {@link #execute() execute} method. - * - */ -@InterfaceAudience.Private -public class IncrementalTableBackupClient extends TableBackupClient { - private static final Log LOG = LogFactory.getLog(IncrementalTableBackupClient.class); - - protected IncrementalTableBackupClient() { - } - - public IncrementalTableBackupClient(final Connection conn, final String backupId, - BackupRequest request) throws IOException { - super(conn, backupId, request); - } - - protected List filterMissingFiles(List incrBackupFileList) throws IOException { - FileSystem fs = FileSystem.get(conf); - List list = new ArrayList(); - for (String file : incrBackupFileList) { - Path p = new Path(file); - if (fs.exists(p) || isActiveWalPath(p)) { - list.add(file); - } else { - LOG.warn("Can't find file: " + file); - } - } - return list; - } - - /** - * Check if a given path is belongs to active WAL directory - * @param p path - * @return true, if yes - */ - protected boolean isActiveWalPath(Path p) { - return !AbstractFSWALProvider.isArchivedLogFile(p); - } - - protected static int getIndex(TableName tbl, List sTableList) { - if (sTableList == null) return 0; - for (int i = 0; i < sTableList.size(); i++) { - if (tbl.equals(sTableList.get(i))) { - return i; - } - } - return -1; - } - - /* - * Reads bulk load records from backup table, iterates through the records and forms the paths - * for bulk loaded hfiles. Copies the bulk loaded hfiles to backup destination - * @param sTableList list of tables to be backed up - * @return map of table to List of files - */ - protected Map>[] handleBulkLoad(List sTableList) throws IOException { - Map>[] mapForSrc = new Map[sTableList.size()]; - List activeFiles = new ArrayList(); - List archiveFiles = new ArrayList(); - Pair>>>>, List> pair = - backupManager.readBulkloadRows(sTableList); - Map>>>> map = pair.getFirst(); - FileSystem fs = FileSystem.get(conf); - FileSystem tgtFs; - try { - tgtFs = FileSystem.get(new URI(backupInfo.getBackupRootDir()), conf); - } catch (URISyntaxException use) { - throw new IOException("Unable to get FileSystem", use); - } - Path rootdir = FSUtils.getRootDir(conf); - Path tgtRoot = new Path(new Path(backupInfo.getBackupRootDir()), backupId); - for (Map.Entry>>>> tblEntry : - map.entrySet()) { - TableName srcTable = tblEntry.getKey(); - - int srcIdx = getIndex(srcTable, sTableList); - if (srcIdx < 0) { - LOG.warn("Couldn't find " + srcTable + " in source table List"); - continue; - } - if (mapForSrc[srcIdx] == null) { - mapForSrc[srcIdx] = new TreeMap>(Bytes.BYTES_COMPARATOR); - } - Path tblDir = FSUtils.getTableDir(rootdir, srcTable); - Path tgtTable = new Path(new Path(tgtRoot, srcTable.getNamespaceAsString()), - srcTable.getQualifierAsString()); - for (Map.Entry>>> regionEntry : - tblEntry.getValue().entrySet()){ - String regionName = regionEntry.getKey(); - Path regionDir = new Path(tblDir, regionName); - // map from family to List of hfiles - for (Map.Entry>> famEntry : - regionEntry.getValue().entrySet()) { - String fam = famEntry.getKey(); - Path famDir = new Path(regionDir, fam); - List files; - if (!mapForSrc[srcIdx].containsKey(fam.getBytes())) { - files = new ArrayList(); - mapForSrc[srcIdx].put(fam.getBytes(), files); - } else { - files = mapForSrc[srcIdx].get(fam.getBytes()); - } - Path archiveDir = HFileArchiveUtil.getStoreArchivePath(conf, srcTable, regionName, fam); - String tblName = srcTable.getQualifierAsString(); - Path tgtFam = new Path(new Path(tgtTable, regionName), fam); - if (!tgtFs.mkdirs(tgtFam)) { - throw new IOException("couldn't create " + tgtFam); - } - for (Pair fileWithState : famEntry.getValue()) { - String file = fileWithState.getFirst(); - int idx = file.lastIndexOf("/"); - String filename = file; - if (idx > 0) { - filename = file.substring(idx+1); - } - Path p = new Path(famDir, filename); - Path tgt = new Path(tgtFam, filename); - Path archive = new Path(archiveDir, filename); - if (fs.exists(p)) { - if (LOG.isTraceEnabled()) { - LOG.trace("found bulk hfile " + file + " in " + famDir + " for " + tblName); - } - if (LOG.isTraceEnabled()) { - LOG.trace("copying " + p + " to " + tgt); - } - activeFiles.add(p.toString()); - } else if (fs.exists(archive)){ - LOG.debug("copying archive " + archive + " to " + tgt); - archiveFiles.add(archive.toString()); - } - files.add(tgt); - } - } - } - } - - copyBulkLoadedFiles(activeFiles, archiveFiles); - - backupManager.writeBulkLoadedFiles(sTableList, mapForSrc); - backupManager.removeBulkLoadedRows(sTableList, pair.getSecond()); - return mapForSrc; - } - - private void copyBulkLoadedFiles(List activeFiles, List archiveFiles) - throws IOException - { - - try { - // Enable special mode of BackupDistCp - conf.setInt(MapReduceBackupCopyJob.NUMBER_OF_LEVELS_TO_PRESERVE_KEY, 5); - // Copy active files - String tgtDest = backupInfo.getBackupRootDir() + Path.SEPARATOR + backupInfo.getBackupId(); - if (activeFiles.size() > 0) { - String[] toCopy = new String[activeFiles.size()]; - activeFiles.toArray(toCopy); - incrementalCopyHFiles(toCopy, tgtDest); - } - if (archiveFiles.size() > 0) { - String[] toCopy = new String[archiveFiles.size()]; - archiveFiles.toArray(toCopy); - incrementalCopyHFiles(toCopy, tgtDest); - } - } finally { - // Disable special mode of BackupDistCp - conf.unset(MapReduceBackupCopyJob.NUMBER_OF_LEVELS_TO_PRESERVE_KEY); - } - - } - - @Override - public void execute() throws IOException { - - try { - // case PREPARE_INCREMENTAL: - beginBackup(backupManager, backupInfo); - backupInfo.setPhase(BackupPhase.PREPARE_INCREMENTAL); - LOG.debug("For incremental backup, current table set is " - + backupManager.getIncrementalBackupTableSet()); - newTimestamps = - ((IncrementalBackupManager) backupManager).getIncrBackupLogFileMap(); - } catch (Exception e) { - // fail the overall backup and return - failBackup(conn, backupInfo, backupManager, e, "Unexpected Exception : ", - BackupType.INCREMENTAL, conf); - throw new IOException(e); - } - - // case INCREMENTAL_COPY: - try { - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); - // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT - convertWALsToHFiles(); - incrementalCopyHFiles(new String[] {getBulkOutputDir().toString()}, backupInfo.getBackupRootDir()); - // Save list of WAL files copied - backupManager.recordWALFiles(backupInfo.getIncrBackupFileList()); - } catch (Exception e) { - String msg = "Unexpected exception in incremental-backup: incremental copy " + backupId; - // fail the overall backup and return - failBackup(conn, backupInfo, backupManager, e, msg, BackupType.INCREMENTAL, conf); - throw new IOException(e); - } - // case INCR_BACKUP_COMPLETE: - // set overall backup status: complete. Here we make sure to complete the backup. - // After this checkpoint, even if entering cancel process, will let the backup finished - try { - // Set the previousTimestampMap which is before this current log roll to the manifest. - HashMap> previousTimestampMap = - backupManager.readLogTimestampMap(); - backupInfo.setIncrTimestampMap(previousTimestampMap); - - // The table list in backupInfo is good for both full backup and incremental backup. - // For incremental backup, it contains the incremental backup table set. - backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); - - HashMap> newTableSetTimestampMap = - backupManager.readLogTimestampMap(); - - Long newStartCode = - BackupUtils.getMinValue(BackupUtils.getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); - - handleBulkLoad(backupInfo.getTableNames()); - // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.INCREMENTAL, conf); - - } catch (IOException e) { - failBackup(conn, backupInfo, backupManager, e, "Unexpected Exception : ", - BackupType.INCREMENTAL, conf); - throw new IOException(e); - } - } - - protected void incrementalCopyHFiles(String[] files, String backupDest) throws IOException { - - try { - LOG.debug("Incremental copy HFiles is starting. dest="+backupDest); - // set overall backup phase: incremental_copy - backupInfo.setPhase(BackupPhase.INCREMENTAL_COPY); - // get incremental backup file list and prepare parms for DistCp - String[] strArr = new String[files.length + 1]; - System.arraycopy(files, 0, strArr, 0, files.length); - strArr[strArr.length - 1] = backupDest; - - String jobname = "Incremental_Backup-HFileCopy-" + backupInfo.getBackupId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Setting incremental copy HFiles job name to : " + jobname); - } - conf.set(JOB_NAME_CONF_KEY, jobname); - - BackupCopyJob copyService = BackupRestoreFactory.getBackupCopyJob(conf); - int res = copyService.copy(backupInfo, backupManager, conf, BackupType.INCREMENTAL, strArr); - if (res != 0) { - LOG.error("Copy incremental HFile files failed with return code: " + res + "."); - throw new IOException("Failed copy from " + StringUtils.join(files, ',') - + " to " + backupDest); - } - LOG.debug("Incremental copy HFiles from " + StringUtils.join(files, ',') - + " to " + backupDest + " finished."); - } finally { - deleteBulkLoadDirectory(); - } - } - - protected void deleteBulkLoadDirectory() throws IOException { - // delete original bulk load directory on method exit - Path path = getBulkOutputDir(); - FileSystem fs = FileSystem.get(conf); - boolean result = fs.delete(path, true); - if (!result) { - LOG.warn("Could not delete " + path); - } - - } - - protected void convertWALsToHFiles() throws IOException { - // get incremental backup file list and prepare parameters for DistCp - List incrBackupFileList = backupInfo.getIncrBackupFileList(); - // Get list of tables in incremental backup set - Set tableSet = backupManager.getIncrementalBackupTableSet(); - // filter missing files out (they have been copied by previous backups) - incrBackupFileList = filterMissingFiles(incrBackupFileList); - for (TableName table : tableSet) { - // Check if table exists - if (tableExists(table, conn)) { - walToHFiles(incrBackupFileList, table); - } else { - LOG.warn("Table " + table + " does not exists. Skipping in WAL converter"); - } - } - } - - - protected boolean tableExists(TableName table, Connection conn) throws IOException { - try (Admin admin = conn.getAdmin();) { - return admin.tableExists(table); - } - } - - protected void walToHFiles(List dirPaths, TableName tableName) throws IOException { - - Tool player = new WALPlayer(); - - // Player reads all files in arbitrary directory structure and creates - // a Map task for each file. We use ';' as separator - // because WAL file names contains ',' - String dirs = StringUtils.join(dirPaths, ';'); - String jobname = "Incremental_Backup-" + backupId + "-" + tableName.getNameAsString(); - - Path bulkOutputPath = getBulkOutputDirForTable(tableName); - conf.set(WALPlayer.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString()); - conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); - conf.set(JOB_NAME_CONF_KEY, jobname); - String[] playerArgs = { dirs, tableName.getNameAsString() }; - - try { - player.setConf(conf); - int result = player.run(playerArgs); - if(result != 0) { - throw new IOException("WAL Player failed"); - } - conf.unset(WALPlayer.INPUT_FILES_SEPARATOR_KEY); - conf.unset(JOB_NAME_CONF_KEY); - } catch (IOException e) { - throw e; - } catch (Exception ee) { - throw new IOException("Can not convert from directory " + dirs - + " (check Hadoop, HBase and WALPlayer M/R job logs) ", ee); - } - } - - protected Path getBulkOutputDirForTable(TableName table) { - Path tablePath = getBulkOutputDir(); - tablePath = new Path(tablePath, table.getNamespaceAsString()); - tablePath = new Path(tablePath, table.getQualifierAsString()); - return new Path(tablePath, "data"); - } - - protected Path getBulkOutputDir() { - String backupId = backupInfo.getBackupId(); - Path path = new Path(backupInfo.getBackupRootDir()); - path = new Path(path, ".tmp"); - path = new Path(path, backupId); - return path; - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java deleted file mode 100644 index fc0fddee379..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java +++ /dev/null @@ -1,283 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.RestoreRequest; -import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.backup.util.RestoreTool; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.LoadQueueItem; - -/** - * Restore table implementation - * - */ -@InterfaceAudience.Private -public class RestoreTablesClient { - private static final Log LOG = LogFactory.getLog(RestoreTablesClient.class); - - private Configuration conf; - private Connection conn; - private String backupId; - private TableName[] sTableArray; - private TableName[] tTableArray; - private String targetRootDir; - private boolean isOverwrite; - - public RestoreTablesClient(Connection conn, RestoreRequest request) throws IOException { - this.targetRootDir = request.getBackupRootDir(); - this.backupId = request.getBackupId(); - this.sTableArray = request.getFromTables(); - this.tTableArray = request.getToTables(); - if (tTableArray == null || tTableArray.length == 0) { - this.tTableArray = sTableArray; - } - this.isOverwrite = request.isOverwrite(); - this.conn = conn; - this.conf = conn.getConfiguration(); - - } - - /** - * Validate target tables - * @param conn connection - * @param mgr table state manager - * @param tTableArray: target tables - * @param isOverwrite overwrite existing table - * @throws IOException exception - */ - private void checkTargetTables(TableName[] tTableArray, boolean isOverwrite) throws IOException { - ArrayList existTableList = new ArrayList<>(); - ArrayList disabledTableList = new ArrayList<>(); - - // check if the tables already exist - try (Admin admin = conn.getAdmin();) { - for (TableName tableName : tTableArray) { - if (admin.tableExists(tableName)) { - existTableList.add(tableName); - if (admin.isTableDisabled(tableName)) { - disabledTableList.add(tableName); - } - } else { - LOG.info("HBase table " + tableName - + " does not exist. It will be created during restore process"); - } - } - } - - if (existTableList.size() > 0) { - if (!isOverwrite) { - LOG.error("Existing table (" + existTableList - + ") found in the restore target, please add " - + "\"-o\" as overwrite option in the command if you mean" - + " to restore to these existing tables"); - throw new IOException("Existing table found in target while no \"-o\" " - + "as overwrite option found"); - } else { - if (disabledTableList.size() > 0) { - LOG.error("Found offline table in the restore target, " - + "please enable them before restore with \"-overwrite\" option"); - LOG.info("Offline table list in restore target: " + disabledTableList); - throw new IOException( - "Found offline table in the target when restore with \"-overwrite\" option"); - } - } - } - } - - /** - * Restore operation handle each backupImage in array - * @param svc: master services - * @param images: array BackupImage - * @param sTable: table to be restored - * @param tTable: table to be restored to - * @param truncateIfExists: truncate table - * @throws IOException exception - */ - - private void restoreImages(BackupImage[] images, TableName sTable, TableName tTable, - boolean truncateIfExists) throws IOException { - - // First image MUST be image of a FULL backup - BackupImage image = images[0]; - String rootDir = image.getRootDir(); - String backupId = image.getBackupId(); - Path backupRoot = new Path(rootDir); - RestoreTool restoreTool = new RestoreTool(conf, backupRoot, backupId); - Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId); - String lastIncrBackupId = images.length == 1 ? null : images[images.length - 1].getBackupId(); - // We need hFS only for full restore (see the code) - BackupManifest manifest = HBackupFileSystem.getManifest(conf, backupRoot, backupId); - if (manifest.getType() == BackupType.FULL) { - LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from full" + " backup image " - + tableBackupPath.toString()); - conf.set(JOB_NAME_CONF_KEY, "Full_Restore-" + backupId + "-" + tTable); - restoreTool.fullRestoreTable(conn, tableBackupPath, sTable, tTable, truncateIfExists, - lastIncrBackupId); - conf.unset(JOB_NAME_CONF_KEY); - } else { // incremental Backup - throw new IOException("Unexpected backup type " + image.getType()); - } - - if (images.length == 1) { - // full backup restore done - return; - } - - List dirList = new ArrayList(); - // add full backup path - // full backup path comes first - for (int i = 1; i < images.length; i++) { - BackupImage im = images[i]; - String fileBackupDir = - HBackupFileSystem.getTableBackupDataDir(im.getRootDir(), im.getBackupId(), sTable); - dirList.add(new Path(fileBackupDir)); - } - - String dirs = StringUtils.join(dirList, ","); - LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from log dirs: " + dirs); - Path[] paths = new Path[dirList.size()]; - dirList.toArray(paths); - conf.set(JOB_NAME_CONF_KEY, "Incremental_Restore-" + backupId + "-" + tTable); - restoreTool.incrementalRestoreTable(conn, tableBackupPath, paths, new TableName[] { sTable }, - new TableName[] { tTable }, lastIncrBackupId); - LOG.info(sTable + " has been successfully restored to " + tTable); - } - - /** - * Restore operation. Stage 2: resolved Backup Image dependency - * @param backupManifestMap : tableName, Manifest - * @param sTableArray The array of tables to be restored - * @param tTableArray The array of mapping tables to restore to - * @return set of BackupImages restored - * @throws IOException exception - */ - private void restore(HashMap backupManifestMap, - TableName[] sTableArray, TableName[] tTableArray, boolean isOverwrite) throws IOException { - TreeSet restoreImageSet = new TreeSet(); - boolean truncateIfExists = isOverwrite; - Set backupIdSet = new HashSet<>(); - - for (int i = 0; i < sTableArray.length; i++) { - TableName table = sTableArray[i]; - - BackupManifest manifest = backupManifestMap.get(table); - // Get the image list of this backup for restore in time order from old - // to new. - List list = new ArrayList(); - list.add(manifest.getBackupImage()); - TreeSet set = new TreeSet(list); - List depList = manifest.getDependentListByTable(table); - set.addAll(depList); - BackupImage[] arr = new BackupImage[set.size()]; - set.toArray(arr); - restoreImages(arr, table, tTableArray[i], truncateIfExists); - restoreImageSet.addAll(list); - if (restoreImageSet != null && !restoreImageSet.isEmpty()) { - LOG.info("Restore includes the following image(s):"); - for (BackupImage image : restoreImageSet) { - LOG.info("Backup: " + image.getBackupId() + " " - + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table)); - if (image.getType() == BackupType.INCREMENTAL) { - backupIdSet.add(image.getBackupId()); - LOG.debug("adding " + image.getBackupId() + " for bulk load"); - } - } - } - } - try (BackupSystemTable table = new BackupSystemTable(conn)) { - List sTableList = Arrays.asList(sTableArray); - for (String id : backupIdSet) { - LOG.debug("restoring bulk load for " + id); - Map>[] mapForSrc = table.readBulkLoadedFiles(id, sTableList); - Map loaderResult; - conf.setBoolean(LoadIncrementalHFiles.ALWAYS_COPY_FILES, true); - LoadIncrementalHFiles loader = BackupUtils.createLoader(conf); - for (int i = 0; i < sTableList.size(); i++) { - if (mapForSrc[i] != null && !mapForSrc[i].isEmpty()) { - loaderResult = loader.run(mapForSrc[i], tTableArray[i]); - LOG.debug("bulk loading " + sTableList.get(i) + " to " + tTableArray[i]); - if (loaderResult.isEmpty()) { - String msg = "Couldn't bulk load for " + sTableList.get(i) + " to " + tTableArray[i]; - LOG.error(msg); - throw new IOException(msg); - } - } - } - } - } - LOG.debug("restoreStage finished"); - } - - static long getTsFromBackupId(String backupId) { - if (backupId == null) { - return 0; - } - return Long.parseLong(backupId.substring(backupId.lastIndexOf("_") + 1)); - } - - static boolean withinRange(long a, long lower, long upper) { - if (a < lower || a > upper) { - return false; - } - return true; - } - - public void execute() throws IOException { - - // case VALIDATION: - // check the target tables - checkTargetTables(tTableArray, isOverwrite); - - // case RESTORE_IMAGES: - HashMap backupManifestMap = new HashMap<>(); - // check and load backup image manifest for the tables - Path rootPath = new Path(targetRootDir); - HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath, - backupId); - - restore(backupManifestMap, sTableArray, tTableArray, isOverwrite); - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java deleted file mode 100644 index aa0ec5f4ee4..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java +++ /dev/null @@ -1,436 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSUtils; - -import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; - -/** - * Base class for backup operation. Concrete implementation for - * full and incremental backup are delegated to corresponding sub-classes: - * {@link FullTableBackupClient} and {@link IncrementalTableBackupClient} - * - */ -@InterfaceAudience.Private -public abstract class TableBackupClient { - - public static final String BACKUP_CLIENT_IMPL_CLASS = "backup.client.impl.class"; - - @VisibleForTesting - public static final String BACKUP_TEST_MODE_STAGE = "backup.test.mode.stage"; - - private static final Log LOG = LogFactory.getLog(TableBackupClient.class); - - protected Configuration conf; - protected Connection conn; - protected String backupId; - protected List tableList; - protected HashMap newTimestamps = null; - - protected BackupManager backupManager; - protected BackupInfo backupInfo; - - public TableBackupClient() { - } - - public TableBackupClient(final Connection conn, final String backupId, BackupRequest request) - throws IOException { - init(conn, backupId, request); - } - - public void init(final Connection conn, final String backupId, BackupRequest request) - throws IOException - { - if (request.getBackupType() == BackupType.FULL) { - backupManager = new BackupManager(conn, conn.getConfiguration()); - } else { - backupManager = new IncrementalBackupManager(conn, conn.getConfiguration()); - } - this.backupId = backupId; - this.tableList = request.getTableList(); - this.conn = conn; - this.conf = conn.getConfiguration(); - backupInfo = - backupManager.createBackupInfo(backupId, request.getBackupType(), tableList, - request.getTargetRootDir(), request.getTotalTasks(), request.getBandwidth()); - if (tableList == null || tableList.isEmpty()) { - this.tableList = new ArrayList<>(backupInfo.getTables()); - } - // Start new session - backupManager.startBackupSession(); - } - - /** - * Begin the overall backup. - * @param backupInfo backup info - * @throws IOException exception - */ - protected void beginBackup(BackupManager backupManager, BackupInfo backupInfo) - throws IOException { - - BackupSystemTable.snapshot(conn); - backupManager.setBackupInfo(backupInfo); - // set the start timestamp of the overall backup - long startTs = EnvironmentEdgeManager.currentTime(); - backupInfo.setStartTs(startTs); - // set overall backup status: ongoing - backupInfo.setState(BackupState.RUNNING); - backupInfo.setPhase(BackupPhase.REQUEST); - LOG.info("Backup " + backupInfo.getBackupId() + " started at " + startTs + "."); - - backupManager.updateBackupInfo(backupInfo); - if (LOG.isDebugEnabled()) { - LOG.debug("Backup session " + backupInfo.getBackupId() + " has been started."); - } - } - - protected String getMessage(Exception e) { - String msg = e.getMessage(); - if (msg == null || msg.equals("")) { - msg = e.getClass().getName(); - } - return msg; - } - - /** - * Delete HBase snapshot for backup. - * @param backupInfo backup info - * @throws Exception exception - */ - protected static void deleteSnapshots(final Connection conn, BackupInfo backupInfo, Configuration conf) - throws IOException { - LOG.debug("Trying to delete snapshot for full backup."); - for (String snapshotName : backupInfo.getSnapshotNames()) { - if (snapshotName == null) { - continue; - } - LOG.debug("Trying to delete snapshot: " + snapshotName); - - try (Admin admin = conn.getAdmin();) { - admin.deleteSnapshot(snapshotName); - } - LOG.debug("Deleting the snapshot " + snapshotName + " for backup " + backupInfo.getBackupId() - + " succeeded."); - } - } - - /** - * Clean up directories with prefix "exportSnapshot-", which are generated when exporting - * snapshots. - * @throws IOException exception - */ - protected static void cleanupExportSnapshotLog(Configuration conf) throws IOException { - FileSystem fs = FSUtils.getCurrentFileSystem(conf); - Path stagingDir = - new Path(conf.get(BackupRestoreConstants.CONF_STAGING_ROOT, fs.getWorkingDirectory() - .toString())); - FileStatus[] files = FSUtils.listStatus(fs, stagingDir); - if (files == null) { - return; - } - for (FileStatus file : files) { - if (file.getPath().getName().startsWith("exportSnapshot-")) { - LOG.debug("Delete log files of exporting snapshot: " + file.getPath().getName()); - if (FSUtils.delete(fs, file.getPath(), true) == false) { - LOG.warn("Can not delete " + file.getPath()); - } - } - } - } - - /** - * Clean up the uncompleted data at target directory if the ongoing backup has already entered - * the copy phase. - */ - protected static void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) { - try { - // clean up the uncompleted data at target directory if the ongoing backup has already entered - // the copy phase - LOG.debug("Trying to cleanup up target dir. Current backup phase: " - + backupInfo.getPhase()); - if (backupInfo.getPhase().equals(BackupPhase.SNAPSHOTCOPY) - || backupInfo.getPhase().equals(BackupPhase.INCREMENTAL_COPY) - || backupInfo.getPhase().equals(BackupPhase.STORE_MANIFEST)) { - FileSystem outputFs = - FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); - - // now treat one backup as a transaction, clean up data that has been partially copied at - // table level - for (TableName table : backupInfo.getTables()) { - Path targetDirPath = - new Path(HBackupFileSystem.getTableBackupDir(backupInfo.getBackupRootDir(), - backupInfo.getBackupId(), table)); - if (outputFs.delete(targetDirPath, true)) { - LOG.debug("Cleaning up uncompleted backup data at " + targetDirPath.toString() - + " done."); - } else { - LOG.debug("No data has been copied to " + targetDirPath.toString() + "."); - } - - Path tableDir = targetDirPath.getParent(); - FileStatus[] backups = FSUtils.listStatus(outputFs, tableDir); - if (backups == null || backups.length == 0) { - outputFs.delete(tableDir, true); - LOG.debug(tableDir.toString() + " is empty, remove it."); - } - } - } - - } catch (IOException e1) { - LOG.error("Cleaning up uncompleted backup data of " + backupInfo.getBackupId() + " at " - + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + "."); - } - } - - /** - * Fail the overall backup. - * @param backupInfo backup info - * @param e exception - * @throws Exception exception - */ - protected void failBackup(Connection conn, BackupInfo backupInfo, BackupManager backupManager, - Exception e, String msg, BackupType type, Configuration conf) throws IOException { - - try { - LOG.error(msg + getMessage(e), e); - // If this is a cancel exception, then we've already cleaned. - // set the failure timestamp of the overall backup - backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); - // set failure message - backupInfo.setFailedMsg(e.getMessage()); - // set overall backup status: failed - backupInfo.setState(BackupState.FAILED); - // compose the backup failed data - String backupFailedData = - "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs() - + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" + backupInfo.getPhase() - + ",failedmessage=" + backupInfo.getFailedMsg(); - LOG.error(backupFailedData); - cleanupAndRestoreBackupSystem(conn, backupInfo, conf); - // If backup session is updated to FAILED state - means we - // processed recovery already. - backupManager.updateBackupInfo(backupInfo); - backupManager.finishBackupSession(); - LOG.error("Backup " + backupInfo.getBackupId() + " failed."); - } catch (IOException ee) { - LOG.error("Please run backup repair tool manually to restore backup system integrity"); - throw ee; - } - } - - public static void cleanupAndRestoreBackupSystem (Connection conn, BackupInfo backupInfo, - Configuration conf) throws IOException - { - BackupType type = backupInfo.getType(); - // if full backup, then delete HBase snapshots if there already are snapshots taken - // and also clean up export snapshot log files if exist - if (type == BackupType.FULL) { - deleteSnapshots(conn, backupInfo, conf); - cleanupExportSnapshotLog(conf); - } - BackupSystemTable.restoreFromSnapshot(conn); - BackupSystemTable.deleteSnapshot(conn); - // clean up the uncompleted data at target directory if the ongoing backup has already entered - // the copy phase - // For incremental backup, DistCp logs will be cleaned with the targetDir. - cleanupTargetDir(backupInfo, conf); - } - - - - /** - * Add manifest for the current backup. The manifest is stored within the table backup directory. - * @param backupInfo The current backup info - * @throws IOException exception - * @throws BackupException exception - */ - protected void addManifest(BackupInfo backupInfo, BackupManager backupManager, BackupType type, - Configuration conf) throws IOException, BackupException { - // set the overall backup phase : store manifest - backupInfo.setPhase(BackupPhase.STORE_MANIFEST); - - BackupManifest manifest; - - // Since we have each table's backup in its own directory structure, - // we'll store its manifest with the table directory. - for (TableName table : backupInfo.getTables()) { - manifest = new BackupManifest(backupInfo, table); - ArrayList ancestors = backupManager.getAncestors(backupInfo, table); - for (BackupImage image : ancestors) { - manifest.addDependentImage(image); - } - - if (type == BackupType.INCREMENTAL) { - // We'll store the log timestamps for this table only in its manifest. - HashMap> tableTimestampMap = - new HashMap>(); - tableTimestampMap.put(table, backupInfo.getIncrTimestampMap().get(table)); - manifest.setIncrTimestampMap(tableTimestampMap); - ArrayList ancestorss = backupManager.getAncestors(backupInfo); - for (BackupImage image : ancestorss) { - manifest.addDependentImage(image); - } - } - manifest.store(conf); - } - - // For incremental backup, we store a overall manifest in - // /WALs/ - // This is used when created the next incremental backup - if (type == BackupType.INCREMENTAL) { - manifest = new BackupManifest(backupInfo); - // set the table region server start and end timestamps for incremental backup - manifest.setIncrTimestampMap(backupInfo.getIncrTimestampMap()); - ArrayList ancestors = backupManager.getAncestors(backupInfo); - for (BackupImage image : ancestors) { - manifest.addDependentImage(image); - } - manifest.store(conf); - } - } - - /** - * Get backup request meta data dir as string. - * @param backupInfo backup info - * @return meta data dir - */ - protected String obtainBackupMetaDataStr(BackupInfo backupInfo) { - StringBuffer sb = new StringBuffer(); - sb.append("type=" + backupInfo.getType() + ",tablelist="); - for (TableName table : backupInfo.getTables()) { - sb.append(table + ";"); - } - if (sb.lastIndexOf(";") > 0) { - sb.delete(sb.lastIndexOf(";"), sb.lastIndexOf(";") + 1); - } - sb.append(",targetRootDir=" + backupInfo.getBackupRootDir()); - - return sb.toString(); - } - - /** - * Clean up directories with prefix "_distcp_logs-", which are generated when DistCp copying - * hlogs. - * @throws IOException exception - */ - protected void cleanupDistCpLog(BackupInfo backupInfo, Configuration conf) throws IOException { - Path rootPath = new Path(backupInfo.getHLogTargetDir()).getParent(); - FileSystem fs = FileSystem.get(rootPath.toUri(), conf); - FileStatus[] files = FSUtils.listStatus(fs, rootPath); - if (files == null) { - return; - } - for (FileStatus file : files) { - if (file.getPath().getName().startsWith("_distcp_logs")) { - LOG.debug("Delete log files of DistCp: " + file.getPath().getName()); - FSUtils.delete(fs, file.getPath(), true); - } - } - } - - /** - * Complete the overall backup. - * @param backupInfo backup info - * @throws Exception exception - */ - protected void completeBackup(final Connection conn, BackupInfo backupInfo, - BackupManager backupManager, BackupType type, Configuration conf) throws IOException { - // set the complete timestamp of the overall backup - backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); - // set overall backup status: complete - backupInfo.setState(BackupState.COMPLETE); - backupInfo.setProgress(100); - // add and store the manifest for the backup - addManifest(backupInfo, backupManager, type, conf); - - // compose the backup complete data - String backupCompleteData = - obtainBackupMetaDataStr(backupInfo) + ",startts=" + backupInfo.getStartTs() - + ",completets=" + backupInfo.getCompleteTs() + ",bytescopied=" - + backupInfo.getTotalBytesCopied(); - if (LOG.isDebugEnabled()) { - LOG.debug("Backup " + backupInfo.getBackupId() + " finished: " + backupCompleteData); - } - - // when full backup is done: - // - delete HBase snapshot - // - clean up directories with prefix "exportSnapshot-", which are generated when exporting - // snapshots - if (type == BackupType.FULL) { - deleteSnapshots(conn, backupInfo, conf); - cleanupExportSnapshotLog(conf); - } else if (type == BackupType.INCREMENTAL) { - cleanupDistCpLog(backupInfo, conf); - } - BackupSystemTable.deleteSnapshot(conn); - backupManager.updateBackupInfo(backupInfo); - - // Finish active session - backupManager.finishBackupSession(); - - LOG.info("Backup " + backupInfo.getBackupId() + " completed."); - } - - /** - * Backup request execution - * @throws IOException - */ - public abstract void execute() throws IOException; - - @VisibleForTesting - protected Stage getTestStage() { - return Stage.valueOf("stage_"+ conf.getInt(BACKUP_TEST_MODE_STAGE, 0)); - } - - @VisibleForTesting - protected void failStageIf(Stage stage) throws IOException { - Stage current = getTestStage(); - if (current == stage) { - throw new IOException("Failed stage " + stage+" in testing"); - } - } - - public static enum Stage { - stage_0, stage_1, stage_2, stage_3, stage_4 - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java deleted file mode 100644 index 90e844264ee..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java +++ /dev/null @@ -1,438 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.mapreduce; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupCopyJob; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.snapshot.ExportSnapshot; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.Counters; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.tools.CopyListingFileStatus; -import org.apache.hadoop.tools.DistCp; -import org.apache.hadoop.tools.DistCpConstants; -import org.apache.hadoop.tools.DistCpOptions; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.zookeeper.KeeperException.NoNodeException; - -/** - * Map-Reduce implementation of {@link BackupCopyJob}. Basically, there are 2 types of copy - * operation: one is copying from snapshot, which bases on extending ExportSnapshot's function, the - * other is copying for incremental log files, which bases on extending DistCp's function. - */ -@InterfaceAudience.Private -public class MapReduceBackupCopyJob implements BackupCopyJob { - public static final String NUMBER_OF_LEVELS_TO_PRESERVE_KEY = "num.levels.preserve"; - private static final Log LOG = LogFactory.getLog(MapReduceBackupCopyJob.class); - - private Configuration conf; - - // Accumulated progress within the whole backup process for the copy operation - private float progressDone = 0.1f; - private long bytesCopied = 0; - private static float INIT_PROGRESS = 0.1f; - - // The percentage of the current copy task within the whole task if multiple time copies are - // needed. The default value is 100%, which means only 1 copy task for the whole. - private float subTaskPercntgInWholeTask = 1f; - - public MapReduceBackupCopyJob() { - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - } - - /** - * Get the current copy task percentage within the whole task if multiple copies are needed. - * @return the current copy task percentage - */ - public float getSubTaskPercntgInWholeTask() { - return subTaskPercntgInWholeTask; - } - - /** - * Set the current copy task percentage within the whole task if multiple copies are needed. Must - * be called before calling - * {@link #copy(BackupInfo, BackupManager, Configuration, BackupType, String[])} - * @param subTaskPercntgInWholeTask The percentage of the copy subtask - */ - public void setSubTaskPercntgInWholeTask(float subTaskPercntgInWholeTask) { - this.subTaskPercntgInWholeTask = subTaskPercntgInWholeTask; - } - - static class SnapshotCopy extends ExportSnapshot { - private BackupInfo backupInfo; - private TableName table; - - public SnapshotCopy(BackupInfo backupInfo, TableName table) { - super(); - this.backupInfo = backupInfo; - this.table = table; - } - - public TableName getTable() { - return this.table; - } - - public BackupInfo getBackupInfo() { - return this.backupInfo; - } - } - - /** - * Update the ongoing backup with new progress. - * @param backupInfo backup info - * @param newProgress progress - * @param bytesCopied bytes copied - * @throws NoNodeException exception - */ - static void updateProgress(BackupInfo backupInfo, BackupManager backupManager, - int newProgress, long bytesCopied) throws IOException { - // compose the new backup progress data, using fake number for now - String backupProgressData = newProgress + "%"; - - backupInfo.setProgress(newProgress); - backupManager.updateBackupInfo(backupInfo); - LOG.debug("Backup progress data \"" + backupProgressData - + "\" has been updated to backup system table for " + backupInfo.getBackupId()); - } - - /** - * Extends DistCp for progress updating to backup system table - * during backup. Using DistCpV2 (MAPREDUCE-2765). - * Simply extend it and override execute() method to get the - * Job reference for progress updating. - * Only the argument "src1, [src2, [...]] dst" is supported, - * no more DistCp options. - */ - - class BackupDistCp extends DistCp { - - private BackupInfo backupInfo; - private BackupManager backupManager; - - public BackupDistCp(Configuration conf, DistCpOptions options, BackupInfo backupInfo, - BackupManager backupManager) throws Exception { - super(conf, options); - this.backupInfo = backupInfo; - this.backupManager = backupManager; - } - - - - @Override - public Job execute() throws Exception { - - // reflection preparation for private methods and fields - Class classDistCp = org.apache.hadoop.tools.DistCp.class; - Method methodCleanup = classDistCp.getDeclaredMethod("cleanup"); - - Field fieldInputOptions = getInputOptionsField(classDistCp); - Field fieldSubmitted = classDistCp.getDeclaredField("submitted"); - - methodCleanup.setAccessible(true); - fieldInputOptions.setAccessible(true); - fieldSubmitted.setAccessible(true); - - // execute() logic starts here - assert fieldInputOptions.get(this) != null; - - Job job = null; - try { - - List srcs = getSourcePaths(fieldInputOptions); - - long totalSrcLgth = 0; - for (Path aSrc : srcs) { - totalSrcLgth += - BackupUtils.getFilesLength(aSrc.getFileSystem(super.getConf()), aSrc); - } - - // Async call - job = super.execute(); - // Update the copy progress to system table every 0.5s if progress value changed - int progressReportFreq = - MapReduceBackupCopyJob.this.getConf().getInt("hbase.backup.progressreport.frequency", - 500); - float lastProgress = progressDone; - while (!job.isComplete()) { - float newProgress = - progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS); - - if (newProgress > lastProgress) { - - BigDecimal progressData = - new BigDecimal(newProgress * 100).setScale(1, BigDecimal.ROUND_HALF_UP); - String newProgressStr = progressData + "%"; - LOG.info("Progress: " + newProgressStr); - updateProgress(backupInfo, backupManager, progressData.intValue(), bytesCopied); - LOG.debug("Backup progress data updated to backup system table: \"Progress: " - + newProgressStr + ".\""); - lastProgress = newProgress; - } - Thread.sleep(progressReportFreq); - } - // update the progress data after copy job complete - float newProgress = - progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS); - BigDecimal progressData = - new BigDecimal(newProgress * 100).setScale(1, BigDecimal.ROUND_HALF_UP); - - String newProgressStr = progressData + "%"; - LOG.info("Progress: " + newProgressStr + " subTask: " + subTaskPercntgInWholeTask - + " mapProgress: " + job.mapProgress()); - - // accumulate the overall backup progress - progressDone = newProgress; - bytesCopied += totalSrcLgth; - - updateProgress(backupInfo, backupManager, progressData.intValue(), bytesCopied); - LOG.debug("Backup progress data updated to backup system table: \"Progress: " - + newProgressStr + " - " + bytesCopied + " bytes copied.\""); - } catch (Throwable t) { - LOG.error(t); - throw t; - } - - String jobID = job.getJobID().toString(); - job.getConfiguration().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID, jobID); - - LOG.debug("DistCp job-id: " + jobID + " completed: " + job.isComplete() + " " - + job.isSuccessful()); - Counters ctrs = job.getCounters(); - LOG.debug(ctrs); - if (job.isComplete() && !job.isSuccessful()) { - throw new Exception("DistCp job-id: " + jobID + " failed"); - } - - return job; - } - - private Field getInputOptionsField(Class classDistCp) throws IOException{ - Field f = null; - try { - f = classDistCp.getDeclaredField("inputOptions"); - } catch(Exception e) { - // Haddop 3 - try { - f = classDistCp.getDeclaredField("context"); - } catch (NoSuchFieldException | SecurityException e1) { - throw new IOException(e1); - } - } - return f; - } - - @SuppressWarnings("unchecked") - private List getSourcePaths(Field fieldInputOptions) throws IOException{ - Object options; - try { - options = fieldInputOptions.get(this); - if (options instanceof DistCpOptions) { - return ((DistCpOptions) options).getSourcePaths(); - } else { - // Hadoop 3 - Class classContext = Class.forName("org.apache.hadoop.tools.DistCpContext"); - Method methodGetSourcePaths = classContext.getDeclaredMethod("getSourcePaths"); - methodGetSourcePaths.setAccessible(true); - - return (List) methodGetSourcePaths.invoke(options); - } - } catch (IllegalArgumentException | IllegalAccessException | - ClassNotFoundException | NoSuchMethodException | - SecurityException | InvocationTargetException e) { - throw new IOException(e); - } - - } - - @Override - protected Path createInputFileListing(Job job) throws IOException { - - if (conf.get(NUMBER_OF_LEVELS_TO_PRESERVE_KEY) == null) { - return super.createInputFileListing(job); - } - long totalBytesExpected = 0; - int totalRecords = 0; - Path fileListingPath = getFileListingPath(); - try (SequenceFile.Writer writer = getWriter(fileListingPath);) { - List srcFiles = getSourceFiles(); - if (srcFiles.size() == 0) { - return fileListingPath; - } - totalRecords = srcFiles.size(); - FileSystem fs = srcFiles.get(0).getFileSystem(conf); - for (Path path : srcFiles) { - FileStatus fst = fs.getFileStatus(path); - totalBytesExpected += fst.getLen(); - Text key = getKey(path); - writer.append(key, new CopyListingFileStatus(fst)); - } - writer.close(); - - // update jobs configuration - - Configuration cfg = job.getConfiguration(); - cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, totalBytesExpected); - cfg.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, fileListingPath.toString()); - cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, totalRecords); - } catch (NoSuchFieldException | SecurityException | IllegalArgumentException - | IllegalAccessException | NoSuchMethodException | ClassNotFoundException - | InvocationTargetException e) { - throw new IOException(e); - } - return fileListingPath; - } - - private Text getKey(Path path) { - int level = conf.getInt(NUMBER_OF_LEVELS_TO_PRESERVE_KEY, 1); - int count = 0; - String relPath = ""; - while (count++ < level) { - relPath = Path.SEPARATOR + path.getName() + relPath; - path = path.getParent(); - } - return new Text(relPath); - } - - private List getSourceFiles() throws NoSuchFieldException, SecurityException, - IllegalArgumentException, IllegalAccessException, NoSuchMethodException, - ClassNotFoundException, InvocationTargetException, IOException { - Field options = null; - try { - options = DistCp.class.getDeclaredField("inputOptions"); - } catch (NoSuchFieldException | SecurityException e) { - options = DistCp.class.getDeclaredField("context"); - } - options.setAccessible(true); - return getSourcePaths(options); - } - - - - private SequenceFile.Writer getWriter(Path pathToListFile) throws IOException { - FileSystem fs = pathToListFile.getFileSystem(conf); - fs.delete(pathToListFile, false); - return SequenceFile.createWriter(conf, SequenceFile.Writer.file(pathToListFile), - SequenceFile.Writer.keyClass(Text.class), - SequenceFile.Writer.valueClass(CopyListingFileStatus.class), - SequenceFile.Writer.compression(SequenceFile.CompressionType.NONE)); - } - - } - - /** - * Do backup copy based on different types. - * @param context The backup info - * @param conf The hadoop configuration - * @param copyType The backup copy type - * @param options Options for customized ExportSnapshot or DistCp - * @throws Exception exception - */ - @Override - public int copy(BackupInfo context, BackupManager backupManager, Configuration conf, - BackupType copyType, String[] options) throws IOException { - int res = 0; - - try { - if (copyType == BackupType.FULL) { - SnapshotCopy snapshotCp = new SnapshotCopy(context, context.getTableBySnapshot(options[1])); - LOG.debug("Doing SNAPSHOT_COPY"); - // Make a new instance of conf to be used by the snapshot copy class. - snapshotCp.setConf(new Configuration(conf)); - res = snapshotCp.run(options); - - } else if (copyType == BackupType.INCREMENTAL) { - LOG.debug("Doing COPY_TYPE_DISTCP"); - setSubTaskPercntgInWholeTask(1f); - - BackupDistCp distcp = - new BackupDistCp(new Configuration(conf), null, context, backupManager); - // Handle a special case where the source file is a single file. - // In this case, distcp will not create the target dir. It just take the - // target as a file name and copy source file to the target (as a file name). - // We need to create the target dir before run distcp. - LOG.debug("DistCp options: " + Arrays.toString(options)); - Path dest = new Path(options[options.length - 1]); - String[] newOptions = new String[options.length + 1]; - System.arraycopy(options, 0, newOptions, 1, options.length); - newOptions[0] = "-async"; // run DisCp in async mode - FileSystem destfs = dest.getFileSystem(conf); - if (!destfs.exists(dest)) { - destfs.mkdirs(dest); - } - res = distcp.run(newOptions); - } - return res; - - } catch (Exception e) { - throw new IOException(e); - } - } - - @Override - public void cancel(String jobId) throws IOException { - JobID id = JobID.forName(jobId); - Cluster cluster = new Cluster(this.getConf()); - try { - Job job = cluster.getJob(id); - if (job == null) { - LOG.error("No job found for " + id); - // should we throw exception - return; - } - if (job.isComplete() || job.isRetired()) { - return; - } - - job.killJob(); - LOG.debug("Killed copy job " + id); - } catch (InterruptedException e) { - throw new IOException(e); - } - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java deleted file mode 100644 index dafc88b5ff7..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java +++ /dev/null @@ -1,321 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.mapreduce; - -import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupMergeJob; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.impl.BackupManifest; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.util.Tool; - -/** - * MapReduce implementation of {@link BackupMergeJob} - * Must be initialized with configuration of a backup destination cluster - * - */ - -@InterfaceAudience.Private -public class MapReduceBackupMergeJob implements BackupMergeJob { - public static final Log LOG = LogFactory.getLog(MapReduceBackupMergeJob.class); - - protected Tool player; - protected Configuration conf; - - public MapReduceBackupMergeJob() { - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - } - - @Override - public void run(String[] backupIds) throws IOException { - String bulkOutputConfKey; - - // TODO : run player on remote cluster - player = new MapReduceHFileSplitterJob(); - bulkOutputConfKey = MapReduceHFileSplitterJob.BULK_OUTPUT_CONF_KEY; - // Player reads all files in arbitrary directory structure and creates - // a Map task for each file - String bids = StringUtils.join(backupIds, ","); - - if (LOG.isDebugEnabled()) { - LOG.debug("Merge backup images " + bids); - } - - List> processedTableList = new ArrayList>(); - boolean finishedTables = false; - Connection conn = ConnectionFactory.createConnection(getConf()); - BackupSystemTable table = new BackupSystemTable(conn); - FileSystem fs = FileSystem.get(getConf()); - - try { - - // Get exclusive lock on backup system - table.startBackupExclusiveOperation(); - // Start merge operation - table.startMergeOperation(backupIds); - - // Select most recent backup id - String mergedBackupId = findMostRecentBackupId(backupIds); - - TableName[] tableNames = getTableNamesInBackupImages(backupIds); - String backupRoot = null; - - BackupInfo bInfo = table.readBackupInfo(backupIds[0]); - backupRoot = bInfo.getBackupRootDir(); - - for (int i = 0; i < tableNames.length; i++) { - - LOG.info("Merge backup images for " + tableNames[i]); - - // Find input directories for table - - Path[] dirPaths = findInputDirectories(fs, backupRoot, tableNames[i], backupIds); - String dirs = StringUtils.join(dirPaths, ","); - Path bulkOutputPath = - BackupUtils.getBulkOutputDir(BackupUtils.getFileNameCompatibleString(tableNames[i]), - getConf(), false); - // Delete content if exists - if (fs.exists(bulkOutputPath)) { - if (!fs.delete(bulkOutputPath, true)) { - LOG.warn("Can not delete: " + bulkOutputPath); - } - } - Configuration conf = getConf(); - conf.set(bulkOutputConfKey, bulkOutputPath.toString()); - String[] playerArgs = { dirs, tableNames[i].getNameAsString() }; - - int result = 0; - - player.setConf(getConf()); - result = player.run(playerArgs); - if (!succeeded(result)) { - throw new IOException("Can not merge backup images for " + dirs - + " (check Hadoop/MR and HBase logs). Player return code =" + result); - } - // Add to processed table list - processedTableList.add(new Pair(tableNames[i], bulkOutputPath)); - LOG.debug("Merge Job finished:" + result); - } - List tableList = toTableNameList(processedTableList); - table.updateProcessedTablesForMerge(tableList); - finishedTables = true; - - // Move data - for (Pair tn : processedTableList) { - moveData(fs, backupRoot, tn.getSecond(), tn.getFirst(), mergedBackupId); - } - - // Delete old data and update manifest - List backupsToDelete = getBackupIdsToDelete(backupIds, mergedBackupId); - deleteBackupImages(backupsToDelete, conn, fs, backupRoot); - updateBackupManifest(backupRoot, mergedBackupId, backupsToDelete); - // Finish merge session - table.finishMergeOperation(); - // Release lock - table.finishBackupExclusiveOperation(); - } catch (RuntimeException e) { - - throw e; - } catch (Exception e) { - LOG.error(e); - if (!finishedTables) { - // cleanup bulk directories and finish merge - // merge MUST be repeated (no need for repair) - cleanupBulkLoadDirs(fs, toPathList(processedTableList)); - table.finishMergeOperation(); - table.finishBackupExclusiveOperation(); - throw new IOException("Backup merge operation failed, you should try it again", e); - } else { - // backup repair must be run - throw new IOException( - "Backup merge operation failed, run backup repair tool to restore system's integrity", - e); - } - } finally { - table.close(); - conn.close(); - } - } - - protected List toPathList(List> processedTableList) { - ArrayList list = new ArrayList(); - for (Pair p : processedTableList) { - list.add(p.getSecond()); - } - return list; - } - - protected List toTableNameList(List> processedTableList) { - ArrayList list = new ArrayList(); - for (Pair p : processedTableList) { - list.add(p.getFirst()); - } - return list; - } - - protected void cleanupBulkLoadDirs(FileSystem fs, List pathList) throws IOException { - for (Path path : pathList) { - - if (!fs.delete(path, true)) { - LOG.warn("Can't delete " + path); - } - } - } - - protected void updateBackupManifest(String backupRoot, String mergedBackupId, - List backupsToDelete) throws IllegalArgumentException, IOException { - - BackupManifest manifest = - HBackupFileSystem.getManifest(conf, new Path(backupRoot), mergedBackupId); - manifest.getBackupImage().removeAncestors(backupsToDelete); - // save back - manifest.store(conf); - - } - - protected void deleteBackupImages(List backupIds, Connection conn, FileSystem fs, - String backupRoot) throws IOException { - - // Delete from backup system table - try (BackupSystemTable table = new BackupSystemTable(conn);) { - for (String backupId : backupIds) { - table.deleteBackupInfo(backupId); - } - } - - // Delete from file system - for (String backupId : backupIds) { - Path backupDirPath = HBackupFileSystem.getBackupPath(backupRoot, backupId); - - if (!fs.delete(backupDirPath, true)) { - LOG.warn("Could not delete " + backupDirPath); - } - } - } - - protected List getBackupIdsToDelete(String[] backupIds, String mergedBackupId) { - List list = new ArrayList(); - for (String id : backupIds) { - if (id.equals(mergedBackupId)) { - continue; - } - list.add(id); - } - return list; - } - - protected void moveData(FileSystem fs, String backupRoot, Path bulkOutputPath, TableName tableName, - String mergedBackupId) throws IllegalArgumentException, IOException { - - Path dest = - new Path(HBackupFileSystem.getTableBackupDataDir(backupRoot, mergedBackupId, tableName)); - - // Delete all in dest - if (!fs.delete(dest, true)) { - throw new IOException("Could not delete " + dest); - } - - FileStatus[] fsts = fs.listStatus(bulkOutputPath); - for (FileStatus fst : fsts) { - if (fst.isDirectory()) { - fs.rename(fst.getPath().getParent(), dest); - } - } - - } - - protected String findMostRecentBackupId(String[] backupIds) { - long recentTimestamp = Long.MIN_VALUE; - for (String backupId : backupIds) { - long ts = Long.parseLong(backupId.split("_")[1]); - if (ts > recentTimestamp) { - recentTimestamp = ts; - } - } - return BackupRestoreConstants.BACKUPID_PREFIX + recentTimestamp; - } - - protected TableName[] getTableNamesInBackupImages(String[] backupIds) throws IOException { - - Set allSet = new HashSet(); - - try (Connection conn = ConnectionFactory.createConnection(conf); - BackupSystemTable table = new BackupSystemTable(conn);) { - for (String backupId : backupIds) { - BackupInfo bInfo = table.readBackupInfo(backupId); - - allSet.addAll(bInfo.getTableNames()); - } - } - - TableName[] ret = new TableName[allSet.size()]; - return allSet.toArray(ret); - } - - protected Path[] findInputDirectories(FileSystem fs, String backupRoot, TableName tableName, - String[] backupIds) throws IOException { - - List dirs = new ArrayList(); - - for (String backupId : backupIds) { - Path fileBackupDirPath = - new Path(HBackupFileSystem.getTableBackupDataDir(backupRoot, backupId, tableName)); - if (fs.exists(fileBackupDirPath)) { - dirs.add(fileBackupDirPath); - } else { - if (LOG.isTraceEnabled()) { - LOG.trace("File: " + fileBackupDirPath + " does not exist."); - } - } - } - Path[] ret = new Path[dirs.size()]; - return dirs.toArray(ret); - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java deleted file mode 100644 index 51a6b1ddd6a..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.mapreduce; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.CellSortReducer; -import org.apache.hadoop.hbase.mapreduce.HFileInputFormat; -import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.MapReduceCell; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * A tool to split HFiles into new region boundaries as a MapReduce job. The tool generates HFiles - * for later bulk importing. - */ -@InterfaceAudience.Private -public class MapReduceHFileSplitterJob extends Configured implements Tool { - private static final Log LOG = LogFactory.getLog(MapReduceHFileSplitterJob.class); - final static String NAME = "HFileSplitterJob"; - public final static String BULK_OUTPUT_CONF_KEY = "hfile.bulk.output"; - public final static String TABLES_KEY = "hfile.input.tables"; - public final static String TABLE_MAP_KEY = "hfile.input.tablesmap"; - private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; - - public MapReduceHFileSplitterJob() { - } - - protected MapReduceHFileSplitterJob(final Configuration c) { - super(c); - } - - /** - * A mapper that just writes out cells. This one can be used together with - * {@link CellSortReducer} - */ - static class HFileCellMapper extends - Mapper { - - @Override - public void map(NullWritable key, Cell value, Context context) - throws IOException, InterruptedException { - context.write(new ImmutableBytesWritable(CellUtil.cloneRow(value)), new MapReduceCell(value)); - } - - @Override - public void setup(Context context) throws IOException { - // do nothing - } - } - - /** - * Sets up the actual job. - * @param args The command line parameters. - * @return The newly created job. - * @throws IOException When setting up the job fails. - */ - public Job createSubmittableJob(String[] args) throws IOException { - Configuration conf = getConf(); - String inputDirs = args[0]; - String tabName = args[1]; - conf.setStrings(TABLES_KEY, tabName); - conf.set(FileInputFormat.INPUT_DIR, inputDirs); - Job job = - Job.getInstance(conf, - conf.get(JOB_NAME_CONF_KEY, NAME + "_" + EnvironmentEdgeManager.currentTime())); - job.setJarByClass(MapReduceHFileSplitterJob.class); - job.setInputFormatClass(HFileInputFormat.class); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); - if (hfileOutPath != null) { - LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs); - TableName tableName = TableName.valueOf(tabName); - job.setMapperClass(HFileCellMapper.class); - job.setReducerClass(CellSortReducer.class); - Path outputDir = new Path(hfileOutPath); - FileOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputValueClass(MapReduceCell.class); - try (Connection conn = ConnectionFactory.createConnection(conf); - Table table = conn.getTable(tableName); - RegionLocator regionLocator = conn.getRegionLocator(tableName)) { - HFileOutputFormat2.configureIncrementalLoad(job, table.getDescriptor(), regionLocator); - } - LOG.debug("success configuring load incremental job"); - - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), - org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions.class); - } else { - throw new IOException("No bulk output directory specified"); - } - return job; - } - - /** - * Print usage - * @param errorMsg Error message. Can be null. - */ - private void usage(final String errorMsg) { - if (errorMsg != null && errorMsg.length() > 0) { - System.err.println("ERROR: " + errorMsg); - } - System.err.println("Usage: " + NAME + " [options] "); - System.err.println("Read all HFile's for
and split them to
region boundaries."); - System.err.println("
table to load.\n"); - System.err.println("To generate HFiles for a bulk data load, pass the option:"); - System.err.println(" -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output"); - System.err.println("Other options:"); - System.err.println(" -D " + JOB_NAME_CONF_KEY - + "=jobName - use the specified mapreduce job name for the HFile splitter"); - System.err.println("For performance also consider the following options:\n" - + " -Dmapreduce.map.speculative=false\n" + " -Dmapreduce.reduce.speculative=false"); - } - - /** - * Main entry point. - * @param args The command line parameters. - * @throws Exception When running the job fails. - */ - public static void main(String[] args) throws Exception { - int ret = ToolRunner.run(new MapReduceHFileSplitterJob(HBaseConfiguration.create()), args); - System.exit(ret); - } - - @Override - public int run(String[] args) throws Exception { - if (args.length < 2) { - usage("Wrong number of arguments: " + args.length); - System.exit(-1); - } - Job job = createSubmittableJob(args); - int result = job.waitForCompletion(true) ? 0 : 1; - return result; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java deleted file mode 100644 index bed61ed3212..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.mapreduce; - -import static org.apache.hadoop.hbase.backup.util.BackupUtils.failed; -import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded; - -import java.io.IOException; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.RestoreJob; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles; -import org.apache.hadoop.util.Tool; - - -/** - * MapReduce implementation of {@link RestoreJob} - * - * For backup restore, it runs {@link MapReduceHFileSplitterJob} job and creates - * HFiles which are aligned with a region boundaries of a table being - * restored. - * - * The resulting HFiles then are loaded using HBase bulk load tool - * {@link LoadIncrementalHFiles} - */ -@InterfaceAudience.Private -public class MapReduceRestoreJob implements RestoreJob { - public static final Log LOG = LogFactory.getLog(MapReduceRestoreJob.class); - - private Tool player; - private Configuration conf; - - public MapReduceRestoreJob() { - } - - @Override - public void run(Path[] dirPaths, TableName[] tableNames, TableName[] newTableNames, - boolean fullBackupRestore) throws IOException { - - String bulkOutputConfKey; - - player = new MapReduceHFileSplitterJob(); - bulkOutputConfKey = MapReduceHFileSplitterJob.BULK_OUTPUT_CONF_KEY; - // Player reads all files in arbitrary directory structure and creates - // a Map task for each file - String dirs = StringUtils.join(dirPaths, ","); - - if (LOG.isDebugEnabled()) { - LOG.debug("Restore " + (fullBackupRestore ? "full" : "incremental") - + " backup from directory " + dirs + " from hbase tables " - + StringUtils.join(tableNames, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND) - + " to tables " - + StringUtils.join(newTableNames, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND)); - } - - for (int i = 0; i < tableNames.length; i++) { - - LOG.info("Restore " + tableNames[i] + " into " + newTableNames[i]); - - Path bulkOutputPath = - BackupUtils.getBulkOutputDir(BackupUtils.getFileNameCompatibleString(newTableNames[i]), - getConf()); - Configuration conf = getConf(); - conf.set(bulkOutputConfKey, bulkOutputPath.toString()); - String[] playerArgs = - { - dirs, - fullBackupRestore ? newTableNames[i].getNameAsString() : tableNames[i] - .getNameAsString() }; - - int result = 0; - int loaderResult = 0; - try { - - player.setConf(getConf()); - result = player.run(playerArgs); - if (succeeded(result)) { - // do bulk load - LoadIncrementalHFiles loader = BackupUtils.createLoader(getConf()); - if (LOG.isDebugEnabled()) { - LOG.debug("Restoring HFiles from directory " + bulkOutputPath); - } - String[] args = { bulkOutputPath.toString(), newTableNames[i].getNameAsString() }; - loaderResult = loader.run(args); - - if (failed(loaderResult)) { - throw new IOException("Can not restore from backup directory " + dirs - + " (check Hadoop and HBase logs). Bulk loader return code =" + loaderResult); - } - } else { - throw new IOException("Can not restore from backup directory " + dirs - + " (check Hadoop/MR and HBase logs). Player return code =" + result); - } - LOG.debug("Restore Job finished:" + result); - } catch (Exception e) { - LOG.error(e); - throw new IOException("Can not restore from backup directory " + dirs - + " (check Hadoop and HBase logs) ", e); - } - } - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java deleted file mode 100644 index 0f1722fd6c6..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ /dev/null @@ -1,143 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.master; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate; - -/** - * Implementation of a log cleaner that checks if a log is still scheduled for incremental backup - * before deleting it when its TTL is over. - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class BackupLogCleaner extends BaseLogCleanerDelegate { - private static final Log LOG = LogFactory.getLog(BackupLogCleaner.class); - - private boolean stopped = false; - private Connection conn; - - public BackupLogCleaner() { - } - - @Override - public void init(Map params) { - if (params != null && params.containsKey(HMaster.MASTER)) { - MasterServices master = (MasterServices) params.get(HMaster.MASTER); - conn = master.getConnection(); - if (getConf() == null) { - super.setConf(conn.getConfiguration()); - } - } - if (conn == null) { - try { - conn = ConnectionFactory.createConnection(getConf()); - } catch (IOException ioe) { - throw new RuntimeException("Failed to create connection", ioe); - } - } - } - - @Override - public Iterable getDeletableFiles(Iterable files) { - // all members of this class are null if backup is disabled, - // so we cannot filter the files - if (this.getConf() == null || !BackupManager.isBackupEnabled(getConf())) { - if (LOG.isDebugEnabled()) { - LOG.debug("Backup is not enabled. Check your " - + BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting"); - } - return files; - } - - List list = new ArrayList(); - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - // If we do not have recorded backup sessions - try { - if (!table.hasBackupSessions()) { - LOG.trace("BackupLogCleaner has no backup sessions"); - return files; - } - } catch (TableNotFoundException tnfe) { - LOG.warn("backup system table is not available" + tnfe.getMessage()); - return files; - } - - for (FileStatus file : files) { - String wal = file.getPath().toString(); - boolean logInSystemTable = table.isWALFileDeletable(wal); - if (LOG.isDebugEnabled()) { - if (logInSystemTable) { - LOG.debug("Found log file in backup system table, deleting: " + wal); - list.add(file); - } else { - LOG.debug("Didn't find this log in backup system table, keeping: " + wal); - } - } - } - return list; - } catch (IOException e) { - LOG.error("Failed to get backup system table table, therefore will keep all files", e); - // nothing to delete - return new ArrayList(); - } - } - - @Override - public void setConf(Configuration config) { - // If backup is disabled, keep all members null - super.setConf(config); - if (!config.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, - BackupRestoreConstants.BACKUP_ENABLE_DEFAULT)) { - LOG.warn("Backup is disabled - allowing all wals to be deleted"); - } - } - - @Override - public void stop(String why) { - if (this.stopped) { - return; - } - this.stopped = true; - LOG.info("Stopping BackupLogCleaner"); - } - - @Override - public boolean isStopped() { - return this.stopped; - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java deleted file mode 100644 index 567d5ec6817..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.master; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ThreadPoolExecutor; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.CoordinatedStateManager; -import org.apache.hadoop.hbase.errorhandling.ForeignException; -import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.MetricsMaster; -import org.apache.hadoop.hbase.procedure.MasterProcedureManager; -import org.apache.hadoop.hbase.procedure.Procedure; -import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; -import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs; -import org.apache.hadoop.hbase.procedure.RegionServerProcedureManager; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameStringPair; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; -import org.apache.zookeeper.KeeperException; - -/** - * Master procedure manager for coordinated cluster-wide WAL roll operation, which is run during - * backup operation, see {@link MasterProcedureManager} and and {@link RegionServerProcedureManager} - */ -@InterfaceAudience.Private -public class LogRollMasterProcedureManager extends MasterProcedureManager { - private static final Log LOG = LogFactory.getLog(LogRollMasterProcedureManager.class); - - public static final String ROLLLOG_PROCEDURE_SIGNATURE = "rolllog-proc"; - public static final String ROLLLOG_PROCEDURE_NAME = "rolllog"; - public static final String BACKUP_WAKE_MILLIS_KEY = "hbase.backup.logroll.wake.millis"; - public static final String BACKUP_TIMEOUT_MILLIS_KEY = "hbase.backup.logroll.timeout.millis"; - public static final String BACKUP_POOL_THREAD_NUMBER_KEY = "hbase.backup.logroll.pool.thread.number"; - - public static final int BACKUP_WAKE_MILLIS_DEFAULT = 500; - public static final int BACKUP_TIMEOUT_MILLIS_DEFAULT = 180000; - public static final int BACKUP_POOL_THREAD_NUMBER_DEFAULT = 8; - private MasterServices master; - private ProcedureCoordinator coordinator; - private boolean done; - - @Override - public void stop(String why) { - LOG.info("stop: " + why); - } - - @Override - public boolean isStopped() { - return false; - } - - @Override - public void initialize(MasterServices master, MetricsMaster metricsMaster) - throws KeeperException, IOException, UnsupportedOperationException { - this.master = master; - this.done = false; - - // setup the default procedure coordinator - String name = master.getServerName().toString(); - - - // get the configuration for the coordinator - Configuration conf = master.getConfiguration(); - long wakeFrequency = conf.getInt(BACKUP_WAKE_MILLIS_KEY, BACKUP_WAKE_MILLIS_DEFAULT); - long timeoutMillis = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY,BACKUP_TIMEOUT_MILLIS_DEFAULT); - int opThreads = conf.getInt(BACKUP_POOL_THREAD_NUMBER_KEY, - BACKUP_POOL_THREAD_NUMBER_DEFAULT); - - // setup the default procedure coordinator - ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads); - CoordinatedStateManager coordManager = new ZkCoordinatedStateManager(master); - ProcedureCoordinatorRpcs comms = - coordManager.getProcedureCoordinatorRpcs(getProcedureSignature(), name); - this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency); - - } - - @Override - public String getProcedureSignature() { - return ROLLLOG_PROCEDURE_SIGNATURE; - } - - @Override - public void execProcedure(ProcedureDescription desc) throws IOException { - if (!isBackupEnabled()) { - LOG.warn("Backup is not enabled. Check your " + BackupRestoreConstants.BACKUP_ENABLE_KEY - + " setting"); - return; - } - this.done = false; - // start the process on the RS - ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getInstance()); - List serverNames = master.getServerManager().getOnlineServersList(); - List servers = new ArrayList(); - for (ServerName sn : serverNames) { - servers.add(sn.toString()); - } - - List conf = desc.getConfigurationList(); - byte[] data = new byte[0]; - if (conf.size() > 0) { - // Get backup root path - data = conf.get(0).getValue().getBytes(); - } - Procedure proc = coordinator.startProcedure(monitor, desc.getInstance(), data, servers); - if (proc == null) { - String msg = "Failed to submit distributed procedure for '" + desc.getInstance() + "'"; - LOG.error(msg); - throw new IOException(msg); - } - - try { - // wait for the procedure to complete. A timer thread is kicked off that should cancel this - // if it takes too long. - proc.waitForCompleted(); - LOG.info("Done waiting - exec procedure for " + desc.getInstance()); - LOG.info("Distributed roll log procedure is successful!"); - this.done = true; - } catch (InterruptedException e) { - ForeignException ee = - new ForeignException("Interrupted while waiting for roll log procdure to finish", e); - monitor.receive(ee); - Thread.currentThread().interrupt(); - } catch (ForeignException e) { - ForeignException ee = - new ForeignException("Exception while waiting for roll log procdure to finish", e); - monitor.receive(ee); - } - monitor.rethrowException(); - } - - private boolean isBackupEnabled() { - return BackupManager.isBackupEnabled(master.getConfiguration()); - } - - @Override - public boolean isProcedureDone(ProcedureDescription desc) throws IOException { - return done; - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java deleted file mode 100644 index 183463dcfd5..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.regionserver; - -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.Callable; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.errorhandling.ForeignException; -import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; -import org.apache.hadoop.hbase.procedure.ProcedureMember; -import org.apache.hadoop.hbase.procedure.Subprocedure; -import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.regionserver.RegionServerServices; -import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * This backup sub-procedure implementation forces a WAL rolling on a RS. - */ -@InterfaceAudience.Private -public class LogRollBackupSubprocedure extends Subprocedure { - private static final Log LOG = LogFactory.getLog(LogRollBackupSubprocedure.class); - - private final RegionServerServices rss; - private final LogRollBackupSubprocedurePool taskManager; - private String backupRoot; - - public LogRollBackupSubprocedure(RegionServerServices rss, ProcedureMember member, - ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout, - LogRollBackupSubprocedurePool taskManager, byte[] data) { - - super(member, LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, errorListener, - wakeFrequency, timeout); - LOG.info("Constructing a LogRollBackupSubprocedure."); - this.rss = rss; - this.taskManager = taskManager; - if (data != null) { - backupRoot = new String(data); - } - } - - /** - * Callable task. TODO. We don't need a thread pool to execute roll log. This can be simplified - * with no use of sub-procedure pool. - */ - class RSRollLogTask implements Callable { - RSRollLogTask() { - } - - @Override - public Void call() throws Exception { - if (LOG.isDebugEnabled()) { - LOG.debug("DRPC started: " + rss.getServerName()); - } - - AbstractFSWAL fsWAL = (AbstractFSWAL) rss.getWAL(null); - long filenum = fsWAL.getFilenum(); - List wals = rss.getWALs(); - long highest = -1; - for (WAL wal : wals) { - if (wal == null) continue; - if (((AbstractFSWAL) wal).getFilenum() > highest) { - highest = ((AbstractFSWAL) wal).getFilenum(); - } - } - - LOG.info("Trying to roll log in backup subprocedure, current log number: " + filenum - + " highest: " + highest + " on " + rss.getServerName()); - ((HRegionServer) rss).getWalRoller().requestRollAll(); - long start = EnvironmentEdgeManager.currentTime(); - while (!((HRegionServer) rss).getWalRoller().walRollFinished()) { - Thread.sleep(20); - } - LOG.debug("log roll took " + (EnvironmentEdgeManager.currentTime() - start)); - LOG.info("After roll log in backup subprocedure, current log number: " + fsWAL.getFilenum() - + " on " + rss.getServerName()); - - Connection connection = rss.getConnection(); - try (final BackupSystemTable table = new BackupSystemTable(connection)) { - // sanity check, good for testing - HashMap serverTimestampMap = - table.readRegionServerLastLogRollResult(backupRoot); - String host = rss.getServerName().getHostname(); - int port = rss.getServerName().getPort(); - String server = host + ":" + port; - Long sts = serverTimestampMap.get(host); - if (sts != null && sts > highest) { - LOG.warn("Won't update server's last roll log result: current=" + sts + " new=" + highest); - return null; - } - // write the log number to backup system table. - table.writeRegionServerLastLogRollResult(server, highest, backupRoot); - return null; - } catch (Exception e) { - LOG.error(e); - throw e; - } - } - } - - private void rolllog() throws ForeignException { - monitor.rethrowException(); - - taskManager.submitTask(new RSRollLogTask()); - monitor.rethrowException(); - - // wait for everything to complete. - taskManager.waitForOutstandingTasks(); - monitor.rethrowException(); - - } - - @Override - public void acquireBarrier() throws ForeignException { - // do nothing, executing in inside barrier step. - } - - /** - * do a log roll. - * @return some bytes - */ - @Override - public byte[] insideBarrier() throws ForeignException { - rolllog(); - return null; - } - - /** - * Cancel threads if they haven't finished. - */ - @Override - public void cleanup(Exception e) { - taskManager.abort("Aborting log roll subprocedure tasks for backup due to error", e); - } - - /** - * Hooray! - */ - public void releaseBarrier() { - // NO OP - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java deleted file mode 100644 index d62dbeec364..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.regionserver; - -import java.io.Closeable; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.DaemonThreadFactory; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.errorhandling.ForeignException; - -/** - * Handle running each of the individual tasks for completing a backup procedure on a region - * server. - */ -@InterfaceAudience.Private -public class LogRollBackupSubprocedurePool implements Closeable, Abortable { - private static final Log LOG = LogFactory.getLog(LogRollBackupSubprocedurePool.class); - - /** Maximum number of concurrent snapshot region tasks that can run concurrently */ - private static final String CONCURENT_BACKUP_TASKS_KEY = "hbase.backup.region.concurrentTasks"; - private static final int DEFAULT_CONCURRENT_BACKUP_TASKS = 3; - - private final ExecutorCompletionService taskPool; - private final ThreadPoolExecutor executor; - private volatile boolean aborted; - private final List> futures = new ArrayList>(); - private final String name; - - public LogRollBackupSubprocedurePool(String name, Configuration conf) { - // configure the executor service - long keepAlive = - conf.getLong(LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_KEY, - LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_DEFAULT); - int threads = conf.getInt(CONCURENT_BACKUP_TASKS_KEY, DEFAULT_CONCURRENT_BACKUP_TASKS); - this.name = name; - executor = - new ThreadPoolExecutor(1, threads, keepAlive, TimeUnit.SECONDS, - new LinkedBlockingQueue(), new DaemonThreadFactory("rs(" + name - + ")-backup-pool")); - taskPool = new ExecutorCompletionService(executor); - } - - /** - * Submit a task to the pool. - */ - public void submitTask(final Callable task) { - Future f = this.taskPool.submit(task); - futures.add(f); - } - - /** - * Wait for all of the currently outstanding tasks submitted via {@link #submitTask(Callable)} - * @return true on success, false otherwise - * @throws ForeignException exception - */ - public boolean waitForOutstandingTasks() throws ForeignException { - LOG.debug("Waiting for backup procedure to finish."); - - try { - for (Future f : futures) { - f.get(); - } - return true; - } catch (InterruptedException e) { - if (aborted) { - throw new ForeignException("Interrupted and found to be aborted while waiting for tasks!", - e); - } - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - if (e.getCause() instanceof ForeignException) { - throw (ForeignException) e.getCause(); - } - throw new ForeignException(name, e.getCause()); - } finally { - // close off remaining tasks - for (Future f : futures) { - if (!f.isDone()) { - f.cancel(true); - } - } - } - return false; - } - - /** - * Attempt to cleanly shutdown any running tasks - allows currently running tasks to cleanly - * finish - */ - @Override - public void close() { - executor.shutdown(); - } - - @Override - public void abort(String why, Throwable e) { - if (this.aborted) { - return; - } - - this.aborted = true; - LOG.warn("Aborting because: " + why, e); - this.executor.shutdownNow(); - } - - @Override - public boolean isAborted() { - return this.aborted; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java deleted file mode 100644 index c499ec7e45f..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.regionserver; - -import java.io.IOException; -import java.util.concurrent.ThreadPoolExecutor; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CoordinatedStateManager; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; -import org.apache.hadoop.hbase.procedure.ProcedureMember; -import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs; -import org.apache.hadoop.hbase.procedure.RegionServerProcedureManager; -import org.apache.hadoop.hbase.procedure.Subprocedure; -import org.apache.hadoop.hbase.procedure.SubprocedureFactory; -import org.apache.hadoop.hbase.regionserver.RegionServerServices; -import org.apache.zookeeper.KeeperException; - -/** - * This manager class handles the work dealing with distributed WAL roll request. - *

- * This provides the mechanism necessary to kick off a backup specific {@link Subprocedure} that is - * responsible by this region server. If any failures occur with the sub-procedure, the manager's - * procedure member notifies the procedure coordinator to abort all others. - *

- * On startup, requires {@link #start()} to be called. - *

- * On shutdown, requires org.apache.hadoop.hbase.procedure.ProcedureMember.close() to be called - */ -@InterfaceAudience.Private -public class LogRollRegionServerProcedureManager extends RegionServerProcedureManager { - - private static final Log LOG = LogFactory.getLog(LogRollRegionServerProcedureManager.class); - - /** Conf key for number of request threads to start backup on region servers */ - public static final String BACKUP_REQUEST_THREADS_KEY = "hbase.backup.region.pool.threads"; - /** # of threads for backup work on the rs. */ - public static final int BACKUP_REQUEST_THREADS_DEFAULT = 10; - - public static final String BACKUP_TIMEOUT_MILLIS_KEY = "hbase.backup.timeout"; - public static final long BACKUP_TIMEOUT_MILLIS_DEFAULT = 60000; - - /** Conf key for millis between checks to see if backup work completed or if there are errors */ - public static final String BACKUP_REQUEST_WAKE_MILLIS_KEY = "hbase.backup.region.wakefrequency"; - /** Default amount of time to check for errors while regions finish backup work */ - private static final long BACKUP_REQUEST_WAKE_MILLIS_DEFAULT = 500; - - private RegionServerServices rss; - private ProcedureMemberRpcs memberRpcs; - private ProcedureMember member; - private boolean started = false; - - /** - * Create a default backup procedure manager - */ - public LogRollRegionServerProcedureManager() { - } - - /** - * Start accepting backup procedure requests. - */ - @Override - public void start() { - if (!BackupManager.isBackupEnabled(rss.getConfiguration())) { - LOG.warn("Backup is not enabled. Check your " + BackupRestoreConstants.BACKUP_ENABLE_KEY - + " setting"); - return; - } - this.memberRpcs.start(rss.getServerName().toString(), member); - started = true; - LOG.info("Started region server backup manager."); - } - - /** - * Close this and all running backup procedure tasks - * @param force forcefully stop all running tasks - * @throws IOException exception - */ - @Override - public void stop(boolean force) throws IOException { - if (!started) { - return; - } - String mode = force ? "abruptly" : "gracefully"; - LOG.info("Stopping RegionServerBackupManager " + mode + "."); - - try { - this.member.close(); - } finally { - this.memberRpcs.close(); - } - } - - /** - * If in a running state, creates the specified subprocedure for handling a backup procedure. - * @return Subprocedure to submit to the ProcedureMemeber. - */ - public Subprocedure buildSubprocedure(byte[] data) { - - // don't run a backup if the parent is stop(ping) - if (rss.isStopping() || rss.isStopped()) { - throw new IllegalStateException("Can't start backup procedure on RS: " + rss.getServerName() - + ", because stopping/stopped!"); - } - - LOG.info("Attempting to run a roll log procedure for backup."); - ForeignExceptionDispatcher errorDispatcher = new ForeignExceptionDispatcher(); - Configuration conf = rss.getConfiguration(); - long timeoutMillis = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY, BACKUP_TIMEOUT_MILLIS_DEFAULT); - long wakeMillis = - conf.getLong(BACKUP_REQUEST_WAKE_MILLIS_KEY, BACKUP_REQUEST_WAKE_MILLIS_DEFAULT); - - LogRollBackupSubprocedurePool taskManager = - new LogRollBackupSubprocedurePool(rss.getServerName().toString(), conf); - return new LogRollBackupSubprocedure(rss, member, errorDispatcher, wakeMillis, timeoutMillis, - taskManager, data); - - } - - /** - * Build the actual backup procedure runner that will do all the 'hard' work - */ - public class BackupSubprocedureBuilder implements SubprocedureFactory { - - @Override - public Subprocedure buildSubprocedure(String name, byte[] data) { - return LogRollRegionServerProcedureManager.this.buildSubprocedure(data); - } - } - - @Override - public void initialize(RegionServerServices rss) throws KeeperException { - this.rss = rss; - if (!BackupManager.isBackupEnabled(rss.getConfiguration())) { - LOG.warn("Backup is not enabled. Check your " + BackupRestoreConstants.BACKUP_ENABLE_KEY - + " setting"); - return; - } - CoordinatedStateManager coordManager = new ZkCoordinatedStateManager(rss); - this.memberRpcs = coordManager - .getProcedureMemberRpcs(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE); - - // read in the backup handler configuration properties - Configuration conf = rss.getConfiguration(); - long keepAlive = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY, BACKUP_TIMEOUT_MILLIS_DEFAULT); - int opThreads = conf.getInt(BACKUP_REQUEST_THREADS_KEY, BACKUP_REQUEST_THREADS_DEFAULT); - // create the actual cohort member - ThreadPoolExecutor pool = - ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive); - this.member = new ProcedureMember(memberRpcs, pool, new BackupSubprocedureBuilder()); - } - - @Override - public String getProcedureSignature() { - return "backup-proc"; - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java deleted file mode 100644 index 47bb12bb76e..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.util; - -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Backup set is a named group of HBase tables, which are managed together by Backup/Restore - * framework. Instead of using list of tables in backup or restore operation, one can use set's name - * instead. - */ -@InterfaceAudience.Private -public class BackupSet { - private final String name; - private final List tables; - - public BackupSet(String name, List tables) { - this.name = name; - this.tables = tables; - } - - public String getName() { - return name; - } - - public List getTables() { - return tables; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(name).append("={"); - sb.append(StringUtils.join(tables, ',')); - sb.append("}"); - return sb.toString(); - } - -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java deleted file mode 100644 index 9d87612e0fe..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ /dev/null @@ -1,747 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.util; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.RestoreRequest; -import org.apache.hadoop.hbase.backup.impl.BackupManifest; -import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSTableDescriptors; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * A collection for methods used by multiple classes to backup HBase tables. - */ -@InterfaceAudience.Private -public final class BackupUtils { - protected static final Log LOG = LogFactory.getLog(BackupUtils.class); - public static final String LOGNAME_SEPARATOR = "."; - public static final int MILLISEC_IN_HOUR = 3600000; - - private BackupUtils() { - throw new AssertionError("Instantiating utility class..."); - } - - /** - * Loop through the RS log timestamp map for the tables, for each RS, find the min timestamp value - * for the RS among the tables. - * @param rsLogTimestampMap timestamp map - * @return the min timestamp of each RS - */ - public static HashMap getRSLogTimestampMins( - HashMap> rsLogTimestampMap) { - - if (rsLogTimestampMap == null || rsLogTimestampMap.isEmpty()) { - return null; - } - - HashMap rsLogTimestampMins = new HashMap(); - HashMap> rsLogTimestampMapByRS = - new HashMap>(); - - for (Entry> tableEntry : rsLogTimestampMap.entrySet()) { - TableName table = tableEntry.getKey(); - HashMap rsLogTimestamp = tableEntry.getValue(); - for (Entry rsEntry : rsLogTimestamp.entrySet()) { - String rs = rsEntry.getKey(); - Long ts = rsEntry.getValue(); - if (!rsLogTimestampMapByRS.containsKey(rs)) { - rsLogTimestampMapByRS.put(rs, new HashMap()); - rsLogTimestampMapByRS.get(rs).put(table, ts); - } else { - rsLogTimestampMapByRS.get(rs).put(table, ts); - } - } - } - - for (Entry> entry : rsLogTimestampMapByRS.entrySet()) { - String rs = entry.getKey(); - rsLogTimestampMins.put(rs, BackupUtils.getMinValue(entry.getValue())); - } - - return rsLogTimestampMins; - } - - /** - * copy out Table RegionInfo into incremental backup image need to consider move this logic into - * HBackupFileSystem - * @param conn connection - * @param backupInfo backup info - * @param conf configuration - * @throws IOException exception - * @throws InterruptedException exception - */ - public static void - copyTableRegionInfo(Connection conn, BackupInfo backupInfo, Configuration conf) - throws IOException, InterruptedException { - Path rootDir = FSUtils.getRootDir(conf); - FileSystem fs = rootDir.getFileSystem(conf); - - // for each table in the table set, copy out the table info and region - // info files in the correct directory structure - for (TableName table : backupInfo.getTables()) { - - if (!MetaTableAccessor.tableExists(conn, table)) { - LOG.warn("Table " + table + " does not exists, skipping it."); - continue; - } - TableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, table); - - // write a copy of descriptor to the target directory - Path target = new Path(backupInfo.getTableBackupDir(table)); - FileSystem targetFs = target.getFileSystem(conf); - FSTableDescriptors descriptors = - new FSTableDescriptors(conf, targetFs, FSUtils.getRootDir(conf)); - descriptors.createTableDescriptorForTableDirectory(target, orig, false); - LOG.debug("Attempting to copy table info for:" + table + " target: " + target - + " descriptor: " + orig); - LOG.debug("Finished copying tableinfo."); - List regions = null; - regions = MetaTableAccessor.getTableRegions(conn, table); - // For each region, write the region info to disk - LOG.debug("Starting to write region info for table " + table); - for (RegionInfo regionInfo : regions) { - Path regionDir = - HRegion.getRegionDir(new Path(backupInfo.getTableBackupDir(table)), regionInfo); - regionDir = new Path(backupInfo.getTableBackupDir(table), regionDir.getName()); - writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo); - } - LOG.debug("Finished writing region info for table " + table); - } - } - - /** - * Write the .regioninfo file on-disk. - */ - public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs, - final Path regionInfoDir, RegionInfo regionInfo) throws IOException { - final byte[] content = RegionInfo.toDelimitedByteArray(regionInfo); - Path regionInfoFile = new Path(regionInfoDir, "." + HConstants.REGIONINFO_QUALIFIER_STR); - // First check to get the permissions - FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); - // Write the RegionInfo file content - FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null); - try { - out.write(content); - } finally { - out.close(); - } - } - - /** - * Parses hostname:port from WAL file path - * @param p path to WAL file - * @return hostname:port - */ - public static String parseHostNameFromLogFile(Path p) { - try { - if (AbstractFSWALProvider.isArchivedLogFile(p)) { - return BackupUtils.parseHostFromOldLog(p); - } else { - ServerName sname = AbstractFSWALProvider.getServerNameFromWALDirectoryName(p); - if (sname != null) { - return sname.getAddress().toString(); - } else { - LOG.error("Skip log file (can't parse): " + p); - return null; - } - } - } catch (Exception e) { - LOG.error("Skip log file (can't parse): " + p, e); - return null; - } - } - - /** - * Returns WAL file name - * @param walFileName WAL file name - * @return WAL file name - * @throws IOException exception - * @throws IllegalArgumentException exception - */ - public static String getUniqueWALFileNamePart(String walFileName) throws IOException { - return getUniqueWALFileNamePart(new Path(walFileName)); - } - - /** - * Returns WAL file name - * @param p WAL file path - * @return WAL file name - * @throws IOException exception - */ - public static String getUniqueWALFileNamePart(Path p) throws IOException { - return p.getName(); - } - - /** - * Get the total length of files under the given directory recursively. - * @param fs The hadoop file system - * @param dir The target directory - * @return the total length of files - * @throws IOException exception - */ - public static long getFilesLength(FileSystem fs, Path dir) throws IOException { - long totalLength = 0; - FileStatus[] files = FSUtils.listStatus(fs, dir); - if (files != null) { - for (FileStatus fileStatus : files) { - if (fileStatus.isDirectory()) { - totalLength += getFilesLength(fs, fileStatus.getPath()); - } else { - totalLength += fileStatus.getLen(); - } - } - } - return totalLength; - } - - /** - * Get list of all old WAL files (WALs and archive) - * @param c configuration - * @param hostTimestampMap {host,timestamp} map - * @return list of WAL files - * @throws IOException exception - */ - public static List getWALFilesOlderThan(final Configuration c, - final HashMap hostTimestampMap) throws IOException { - Path rootDir = FSUtils.getRootDir(c); - Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); - Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); - List logFiles = new ArrayList(); - - PathFilter filter = new PathFilter() { - - @Override - public boolean accept(Path p) { - try { - if (AbstractFSWALProvider.isMetaFile(p)) { - return false; - } - String host = parseHostNameFromLogFile(p); - if (host == null) { - return false; - } - Long oldTimestamp = hostTimestampMap.get(host); - Long currentLogTS = BackupUtils.getCreationTime(p); - return currentLogTS <= oldTimestamp; - } catch (Exception e) { - LOG.warn("Can not parse" + p, e); - return false; - } - } - }; - FileSystem fs = FileSystem.get(c); - logFiles = BackupUtils.getFiles(fs, logDir, logFiles, filter); - logFiles = BackupUtils.getFiles(fs, oldLogDir, logFiles, filter); - return logFiles; - } - - public static TableName[] parseTableNames(String tables) { - if (tables == null) { - return null; - } - String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); - - TableName[] ret = new TableName[tableArray.length]; - for (int i = 0; i < tableArray.length; i++) { - ret[i] = TableName.valueOf(tableArray[i]); - } - return ret; - } - - /** - * Check whether the backup path exist - * @param backupStr backup - * @param conf configuration - * @return Yes if path exists - * @throws IOException exception - */ - public static boolean checkPathExist(String backupStr, Configuration conf) throws IOException { - boolean isExist = false; - Path backupPath = new Path(backupStr); - FileSystem fileSys = backupPath.getFileSystem(conf); - String targetFsScheme = fileSys.getUri().getScheme(); - if (LOG.isTraceEnabled()) { - LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme); - } - if (fileSys.exists(backupPath)) { - isExist = true; - } - return isExist; - } - - /** - * Check target path first, confirm it doesn't exist before backup - * @param backupRootPath backup destination path - * @param conf configuration - * @throws IOException exception - */ - public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException { - boolean targetExists = false; - try { - targetExists = checkPathExist(backupRootPath, conf); - } catch (IOException e) { - String expMsg = e.getMessage(); - String newMsg = null; - if (expMsg.contains("No FileSystem for scheme")) { - newMsg = - "Unsupported filesystem scheme found in the backup target url. Error Message: " - + newMsg; - LOG.error(newMsg); - throw new IOException(newMsg); - } else { - throw e; - } - } - - if (targetExists) { - LOG.info("Using existing backup root dir: " + backupRootPath); - } else { - LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created."); - } - } - - /** - * Get the min value for all the Values a map. - * @param map map - * @return the min value - */ - public static Long getMinValue(HashMap map) { - Long minTimestamp = null; - if (map != null) { - ArrayList timestampList = new ArrayList(map.values()); - Collections.sort(timestampList); - // The min among all the RS log timestamps will be kept in backup system table table. - minTimestamp = timestampList.get(0); - } - return minTimestamp; - } - - /** - * Parses host name:port from archived WAL path - * @param p path - * @return host name - * @throws IOException exception - */ - public static String parseHostFromOldLog(Path p) { - try { - String n = p.getName(); - int idx = n.lastIndexOf(LOGNAME_SEPARATOR); - String s = URLDecoder.decode(n.substring(0, idx), "UTF8"); - return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s); - } catch (Exception e) { - LOG.warn("Skip log file (can't parse): " + p); - return null; - } - } - - /** - * Given the log file, parse the timestamp from the file name. The timestamp is the last number. - * @param p a path to the log file - * @return the timestamp - * @throws IOException exception - */ - public static Long getCreationTime(Path p) throws IOException { - int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR); - if (idx < 0) { - throw new IOException("Cannot parse timestamp from path " + p); - } - String ts = p.getName().substring(idx + 1); - return Long.parseLong(ts); - } - - public static List getFiles(FileSystem fs, Path rootDir, List files, - PathFilter filter) throws FileNotFoundException, IOException { - RemoteIterator it = fs.listFiles(rootDir, true); - - while (it.hasNext()) { - LocatedFileStatus lfs = it.next(); - if (lfs.isDirectory()) { - continue; - } - // apply filter - if (filter.accept(lfs.getPath())) { - files.add(lfs.getPath().toString()); - } - } - return files; - } - - public static void cleanupBackupData(BackupInfo context, Configuration conf) throws IOException { - cleanupHLogDir(context, conf); - cleanupTargetDir(context, conf); - } - - /** - * Clean up directories which are generated when DistCp copying hlogs - * @param backupInfo backup info - * @param conf configuration - * @throws IOException exception - */ - private static void cleanupHLogDir(BackupInfo backupInfo, Configuration conf) throws IOException { - - String logDir = backupInfo.getHLogTargetDir(); - if (logDir == null) { - LOG.warn("No log directory specified for " + backupInfo.getBackupId()); - return; - } - - Path rootPath = new Path(logDir).getParent(); - FileSystem fs = FileSystem.get(rootPath.toUri(), conf); - FileStatus[] files = listStatus(fs, rootPath, null); - if (files == null) { - return; - } - for (FileStatus file : files) { - LOG.debug("Delete log files: " + file.getPath().getName()); - fs.delete(file.getPath(), true); - } - } - - private static void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) { - try { - // clean up the data at target directory - LOG.debug("Trying to cleanup up target dir : " + backupInfo.getBackupId()); - String targetDir = backupInfo.getBackupRootDir(); - if (targetDir == null) { - LOG.warn("No target directory specified for " + backupInfo.getBackupId()); - return; - } - - FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); - - for (TableName table : backupInfo.getTables()) { - Path targetDirPath = - new Path(getTableBackupDir(backupInfo.getBackupRootDir(), backupInfo.getBackupId(), - table)); - if (outputFs.delete(targetDirPath, true)) { - LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); - } else { - LOG.info("No data has been found in " + targetDirPath.toString() + "."); - } - - Path tableDir = targetDirPath.getParent(); - FileStatus[] backups = listStatus(outputFs, tableDir, null); - if (backups == null || backups.length == 0) { - outputFs.delete(tableDir, true); - LOG.debug(tableDir.toString() + " is empty, remove it."); - } - } - outputFs.delete(new Path(targetDir, backupInfo.getBackupId()), true); - } catch (IOException e1) { - LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " at " - + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + "."); - } - } - - /** - * Given the backup root dir, backup id and the table name, return the backup image location, - * which is also where the backup manifest file is. return value look like: - * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/backup_1396650096738/default/t1_dn/" - * @param backupRootDir backup root directory - * @param backupId backup id - * @param tableName table name - * @return backupPath String for the particular table - */ - public static String - getTableBackupDir(String backupRootDir, String backupId, TableName tableName) { - return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR - + tableName.getNamespaceAsString() + Path.SEPARATOR + tableName.getQualifierAsString() - + Path.SEPARATOR; - } - - /** - * Sort history list by start time in descending order. - * @param historyList history list - * @return sorted list of BackupCompleteData - */ - public static ArrayList sortHistoryListDesc(ArrayList historyList) { - ArrayList list = new ArrayList(); - TreeMap map = new TreeMap(); - for (BackupInfo h : historyList) { - map.put(Long.toString(h.getStartTs()), h); - } - Iterator i = map.descendingKeySet().iterator(); - while (i.hasNext()) { - list.add(map.get(i.next())); - } - return list; - } - - /** - * Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates - * differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and - * return an empty FileStatus[] while Hadoop 2 will throw FileNotFoundException. - * @param fs file system - * @param dir directory - * @param filter path filter - * @return null if dir is empty or doesn't exist, otherwise FileStatus array - */ - public static FileStatus[] - listStatus(final FileSystem fs, final Path dir, final PathFilter filter) throws IOException { - FileStatus[] status = null; - try { - status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter); - } catch (FileNotFoundException fnfe) { - // if directory doesn't exist, return null - if (LOG.isTraceEnabled()) { - LOG.trace(dir + " doesn't exist"); - } - } - if (status == null || status.length < 1) return null; - return status; - } - - /** - * Return the 'path' component of a Path. In Hadoop, Path is an URI. This method returns the - * 'path' component of a Path's URI: e.g. If a Path is - * hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir, this method returns - * /hbase_trunk/TestTable/compaction.dir. This method is useful if you want to print - * out a Path without qualifying Filesystem instance. - * @param p file system Path whose 'path' component we are to return. - * @return Path portion of the Filesystem - */ - public static String getPath(Path p) { - return p.toUri().getPath(); - } - - /** - * Given the backup root dir and the backup id, return the log file location for an incremental - * backup. - * @param backupRootDir backup root directory - * @param backupId backup id - * @return logBackupDir: ".../user/biadmin/backup1/WALs/backup_1396650096738" - */ - public static String getLogBackupDir(String backupRootDir, String backupId) { - return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR - + HConstants.HREGION_LOGDIR_NAME; - } - - private static List getHistory(Configuration conf, Path backupRootPath) - throws IOException { - // Get all (n) history from backup root destination - FileSystem fs = FileSystem.get(conf); - RemoteIterator it = fs.listLocatedStatus(backupRootPath); - - List infos = new ArrayList(); - while (it.hasNext()) { - LocatedFileStatus lfs = it.next(); - if (!lfs.isDirectory()) continue; - String backupId = lfs.getPath().getName(); - try { - BackupInfo info = loadBackupInfo(backupRootPath, backupId, fs); - infos.add(info); - } catch (IOException e) { - LOG.error("Can not load backup info from: " + lfs.getPath(), e); - } - } - // Sort - Collections.sort(infos, new Comparator() { - - @Override - public int compare(BackupInfo o1, BackupInfo o2) { - long ts1 = getTimestamp(o1.getBackupId()); - long ts2 = getTimestamp(o2.getBackupId()); - if (ts1 == ts2) return 0; - return ts1 < ts2 ? 1 : -1; - } - - private long getTimestamp(String backupId) { - String[] split = backupId.split("_"); - return Long.parseLong(split[1]); - } - }); - return infos; - } - - public static List getHistory(Configuration conf, int n, Path backupRootPath, - BackupInfo.Filter... filters) throws IOException { - List infos = getHistory(conf, backupRootPath); - List ret = new ArrayList(); - for (BackupInfo info : infos) { - if (ret.size() == n) { - break; - } - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(info)) { - passed = false; - break; - } - } - if (passed) { - ret.add(info); - } - } - return ret; - } - - public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs) - throws IOException { - Path backupPath = new Path(backupRootPath, backupId); - - RemoteIterator it = fs.listFiles(backupPath, true); - while (it.hasNext()) { - LocatedFileStatus lfs = it.next(); - if (lfs.getPath().getName().equals(BackupManifest.MANIFEST_FILE_NAME)) { - // Load BackupManifest - BackupManifest manifest = new BackupManifest(fs, lfs.getPath().getParent()); - BackupInfo info = manifest.toBackupInfo(); - return info; - } - } - return null; - } - - /** - * Create restore request. - * @param backupRootDir backup root dir - * @param backupId backup id - * @param check check only - * @param fromTables table list from - * @param toTables table list to - * @param isOverwrite overwrite data - * @return request obkect - */ - public static RestoreRequest createRestoreRequest(String backupRootDir, String backupId, - boolean check, TableName[] fromTables, TableName[] toTables, boolean isOverwrite) { - RestoreRequest.Builder builder = new RestoreRequest.Builder(); - RestoreRequest request = - builder.withBackupRootDir(backupRootDir).withBackupId(backupId).withCheck(check) - .withFromTables(fromTables).withToTables(toTables).withOvewrite(isOverwrite).build(); - return request; - } - - public static boolean validate(HashMap backupManifestMap, - Configuration conf) throws IOException { - boolean isValid = true; - - for (Entry manifestEntry : backupManifestMap.entrySet()) { - TableName table = manifestEntry.getKey(); - TreeSet imageSet = new TreeSet(); - - ArrayList depList = manifestEntry.getValue().getDependentListByTable(table); - if (depList != null && !depList.isEmpty()) { - imageSet.addAll(depList); - } - - LOG.info("Dependent image(s) from old to new:"); - for (BackupImage image : imageSet) { - String imageDir = - HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table); - if (!BackupUtils.checkPathExist(imageDir, conf)) { - LOG.error("ERROR: backup image does not exist: " + imageDir); - isValid = false; - break; - } - LOG.info("Backup image: " + image.getBackupId() + " for '" + table + "' is available"); - } - } - return isValid; - } - - public static Path getBulkOutputDir(String tableName, Configuration conf, boolean deleteOnExit) - throws IOException { - FileSystem fs = FileSystem.get(conf); - String tmp = - conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY); - Path path = - new Path(tmp + Path.SEPARATOR + "bulk_output-" + tableName + "-" - + EnvironmentEdgeManager.currentTime()); - if (deleteOnExit) { - fs.deleteOnExit(path); - } - return path; - } - - public static Path getBulkOutputDir(String tableName, Configuration conf) throws IOException { - return getBulkOutputDir(tableName, conf, true); - } - - public static String getFileNameCompatibleString(TableName table) { - return table.getNamespaceAsString() + "-" + table.getQualifierAsString(); - } - - public static boolean failed(int result) { - return result != 0; - } - - public static boolean succeeded(int result) { - return result == 0; - } - - public static LoadIncrementalHFiles createLoader(Configuration config) throws IOException { - // set configuration for restore: - // LoadIncrementalHFile needs more time - // hbase.rpc.timeout 600000 - // calculates - Configuration conf = new Configuration(config); - conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, MILLISEC_IN_HOUR); - - // By default, it is 32 and loader will fail if # of files in any region exceed this - // limit. Bad for snapshot restore. - conf.setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, Integer.MAX_VALUE); - conf.set(LoadIncrementalHFiles.IGNORE_UNMATCHED_CF_CONF_KEY, "yes"); - LoadIncrementalHFiles loader = null; - try { - loader = new LoadIncrementalHFiles(conf); - } catch (Exception e) { - throw new IOException(e); - } - return loader; - } -} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java deleted file mode 100644 index b00351b12e0..00000000000 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java +++ /dev/null @@ -1,518 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.util; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.TreeMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.RestoreJob; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSTableDescriptors; - -/** - * A collection for methods used by multiple classes to restore HBase tables. - */ -@InterfaceAudience.Private -public class RestoreTool { - - public static final Log LOG = LogFactory.getLog(BackupUtils.class); - private final static long TABLE_AVAILABILITY_WAIT_TIME = 180000; - - private final String[] ignoreDirs = { HConstants.RECOVERED_EDITS_DIR }; - protected Configuration conf = null; - protected Path backupRootPath; - protected String backupId; - protected FileSystem fs; - - // store table name and snapshot dir mapping - private final HashMap snapshotMap = new HashMap<>(); - - public RestoreTool(Configuration conf, final Path backupRootPath, final String backupId) - throws IOException { - this.conf = conf; - this.backupRootPath = backupRootPath; - this.backupId = backupId; - this.fs = backupRootPath.getFileSystem(conf); - } - - /** - * return value represent path for: - * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/archive/data/default/t1_dn" - * @param tableName table name - * @return path to table archive - * @throws IOException exception - */ - Path getTableArchivePath(TableName tableName) throws IOException { - - Path baseDir = - new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId), - HConstants.HFILE_ARCHIVE_DIRECTORY); - Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR); - Path archivePath = new Path(dataDir, tableName.getNamespaceAsString()); - Path tableArchivePath = new Path(archivePath, tableName.getQualifierAsString()); - if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) { - LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists"); - tableArchivePath = null; // empty table has no archive - } - return tableArchivePath; - } - - /** - * Gets region list - * @param tableName table name - * @return RegionList region list - * @throws FileNotFoundException exception - * @throws IOException exception - */ - ArrayList getRegionList(TableName tableName) throws FileNotFoundException, IOException { - Path tableArchivePath = getTableArchivePath(tableName); - ArrayList regionDirList = new ArrayList(); - FileStatus[] children = fs.listStatus(tableArchivePath); - for (FileStatus childStatus : children) { - // here child refer to each region(Name) - Path child = childStatus.getPath(); - regionDirList.add(child); - } - return regionDirList; - } - - - void modifyTableSync(Connection conn, TableDescriptor desc) throws IOException { - - try (Admin admin = conn.getAdmin();) { - admin.modifyTable(desc); - int attempt = 0; - int maxAttempts = 600; - while (!admin.isTableAvailable(desc.getTableName())) { - Thread.sleep(100); - attempt++; - if (attempt++ > maxAttempts) { - throw new IOException("Timeout expired " + (maxAttempts * 100) + "ms"); - } - } - } catch (Exception e) { - throw new IOException(e); - } - } - - /** - * During incremental backup operation. Call WalPlayer to replay WAL in backup image Currently - * tableNames and newTablesNames only contain single table, will be expanded to multiple tables in - * the future - * @param conn HBase connection - * @param tableBackupPath backup path - * @param logDirs : incremental backup folders, which contains WAL - * @param tableNames : source tableNames(table names were backuped) - * @param newTableNames : target tableNames(table names to be restored to) - * @param incrBackupId incremental backup Id - * @throws IOException exception - */ - public void incrementalRestoreTable(Connection conn, Path tableBackupPath, Path[] logDirs, - TableName[] tableNames, TableName[] newTableNames, String incrBackupId) throws IOException { - - try (Admin admin = conn.getAdmin();) { - if (tableNames.length != newTableNames.length) { - throw new IOException("Number of source tables and target tables does not match!"); - } - FileSystem fileSys = tableBackupPath.getFileSystem(this.conf); - - // for incremental backup image, expect the table already created either by user or previous - // full backup. Here, check that all new tables exists - for (TableName tableName : newTableNames) { - if (!admin.tableExists(tableName)) { - throw new IOException("HBase table " + tableName - + " does not exist. Create the table first, e.g. by restoring a full backup."); - } - } - // adjust table schema - for (int i = 0; i < tableNames.length; i++) { - TableName tableName = tableNames[i]; - TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, incrBackupId); - LOG.debug("Found descriptor " + tableDescriptor + " through " + incrBackupId); - - TableName newTableName = newTableNames[i]; - TableDescriptor newTableDescriptor = admin.getDescriptor(newTableName); - List families = Arrays.asList(tableDescriptor.getColumnFamilies()); - List existingFamilies = - Arrays.asList(newTableDescriptor.getColumnFamilies()); - TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(newTableDescriptor); - boolean schemaChangeNeeded = false; - for (ColumnFamilyDescriptor family : families) { - if (!existingFamilies.contains(family)) { - builder.addColumnFamily(family); - schemaChangeNeeded = true; - } - } - for (ColumnFamilyDescriptor family : existingFamilies) { - if (!families.contains(family)) { - builder.removeColumnFamily(family.getName()); - schemaChangeNeeded = true; - } - } - if (schemaChangeNeeded) { - modifyTableSync(conn, builder.build()); - LOG.info("Changed " + newTableDescriptor.getTableName() + " to: " + newTableDescriptor); - } - } - RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf); - - restoreService.run(logDirs, tableNames, newTableNames, false); - } - } - - public void fullRestoreTable(Connection conn, Path tableBackupPath, TableName tableName, - TableName newTableName, boolean truncateIfExists, String lastIncrBackupId) - throws IOException { - createAndRestoreTable(conn, tableName, newTableName, tableBackupPath, truncateIfExists, - lastIncrBackupId); - } - - /** - * Returns value represent path for path to backup table snapshot directory: - * "/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot" - * @param backupRootPath backup root path - * @param tableName table name - * @param backupId backup Id - * @return path for snapshot - */ - Path getTableSnapshotPath(Path backupRootPath, TableName tableName, String backupId) { - return new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId), - HConstants.SNAPSHOT_DIR_NAME); - } - - /** - * Returns value represent path for: - * ""/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot/snapshot_1396650097621_namespace_table" - * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo, - * .data.manifest (trunk) - * @param tableName table name - * @return path to table info - * @throws FileNotFoundException exception - * @throws IOException exception - */ - Path getTableInfoPath(TableName tableName) throws FileNotFoundException, IOException { - Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); - Path tableInfoPath = null; - - // can't build the path directly as the timestamp values are different - FileStatus[] snapshots = fs.listStatus(tableSnapShotPath); - for (FileStatus snapshot : snapshots) { - tableInfoPath = snapshot.getPath(); - // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest"; - if (tableInfoPath.getName().endsWith("data.manifest")) { - break; - } - } - return tableInfoPath; - } - - /** - * Get table descriptor - * @param tableName is the table backed up - * @return {@link TableDescriptor} saved in backup image of the table - */ - TableDescriptor getTableDesc(TableName tableName) throws FileNotFoundException, IOException { - Path tableInfoPath = this.getTableInfoPath(tableName); - SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath); - SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc); - TableDescriptor tableDescriptor = manifest.getTableDescriptor(); - if (!tableDescriptor.getTableName().equals(tableName)) { - LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " - + tableInfoPath.toString()); - LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getTableName().getNameAsString()); - throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName - + " under tableInfoPath: " + tableInfoPath.toString()); - } - return tableDescriptor; - } - - private TableDescriptor getTableDescriptor(FileSystem fileSys, TableName tableName, - String lastIncrBackupId) throws IOException { - if (lastIncrBackupId != null) { - String target = - BackupUtils.getTableBackupDir(backupRootPath.toString(), - lastIncrBackupId, tableName); - return FSTableDescriptors.getTableDescriptorFromFs(fileSys, new Path(target)); - } - return null; - } - - private void createAndRestoreTable(Connection conn, TableName tableName, TableName newTableName, - Path tableBackupPath, boolean truncateIfExists, String lastIncrBackupId) throws IOException { - if (newTableName == null) { - newTableName = tableName; - } - FileSystem fileSys = tableBackupPath.getFileSystem(this.conf); - - // get table descriptor first - TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId); - if (tableDescriptor != null) { - LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId); - } - - if (tableDescriptor == null) { - Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); - if (fileSys.exists(tableSnapshotPath)) { - // snapshot path exist means the backup path is in HDFS - // check whether snapshot dir already recorded for target table - if (snapshotMap.get(tableName) != null) { - SnapshotDescription desc = - SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath); - SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc); - tableDescriptor = manifest.getTableDescriptor(); - } else { - tableDescriptor = getTableDesc(tableName); - snapshotMap.put(tableName, getTableInfoPath(tableName)); - } - if (tableDescriptor == null) { - LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost"); - } - } else { - throw new IOException("Table snapshot directory: " + - tableSnapshotPath + " does not exist."); - } - } - - Path tableArchivePath = getTableArchivePath(tableName); - if (tableArchivePath == null) { - if (tableDescriptor != null) { - // find table descriptor but no archive dir means the table is empty, create table and exit - if (LOG.isDebugEnabled()) { - LOG.debug("find table descriptor but no archive dir for table " + tableName - + ", will only create table"); - } - tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor); - checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, null, tableDescriptor, - truncateIfExists); - return; - } else { - throw new IllegalStateException("Cannot restore hbase table because directory '" - + " tableArchivePath is null."); - } - } - - if (tableDescriptor == null) { - tableDescriptor = TableDescriptorBuilder.newBuilder(newTableName).build(); - } else { - tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor); - } - - // record all region dirs: - // load all files in dir - try { - ArrayList regionPathList = getRegionList(tableName); - - // should only try to create the table with all region informations, so we could pre-split - // the regions in fine grain - checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, regionPathList, - tableDescriptor, truncateIfExists); - RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf); - Path[] paths = new Path[regionPathList.size()]; - regionPathList.toArray(paths); - restoreService.run(paths, new TableName[]{tableName}, new TableName[] {newTableName}, true); - - } catch (Exception e) { - LOG.error(e); - throw new IllegalStateException("Cannot restore hbase table", e); - } - } - - /** - * Gets region list - * @param tableArchivePath table archive path - * @return RegionList region list - * @throws FileNotFoundException exception - * @throws IOException exception - */ - ArrayList getRegionList(Path tableArchivePath) throws FileNotFoundException, IOException { - ArrayList regionDirList = new ArrayList(); - FileStatus[] children = fs.listStatus(tableArchivePath); - for (FileStatus childStatus : children) { - // here child refer to each region(Name) - Path child = childStatus.getPath(); - regionDirList.add(child); - } - return regionDirList; - } - - /** - * Calculate region boundaries and add all the column families to the table descriptor - * @param regionDirList region dir list - * @return a set of keys to store the boundaries - */ - byte[][] generateBoundaryKeys(ArrayList regionDirList) throws FileNotFoundException, - IOException { - TreeMap map = new TreeMap(Bytes.BYTES_COMPARATOR); - // Build a set of keys to store the boundaries - // calculate region boundaries and add all the column families to the table descriptor - for (Path regionDir : regionDirList) { - LOG.debug("Parsing region dir: " + regionDir); - Path hfofDir = regionDir; - - if (!fs.exists(hfofDir)) { - LOG.warn("HFileOutputFormat dir " + hfofDir + " not found"); - } - - FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); - if (familyDirStatuses == null) { - throw new IOException("No families found in " + hfofDir); - } - - for (FileStatus stat : familyDirStatuses) { - if (!stat.isDirectory()) { - LOG.warn("Skipping non-directory " + stat.getPath()); - continue; - } - boolean isIgnore = false; - String pathName = stat.getPath().getName(); - for (String ignore : ignoreDirs) { - if (pathName.contains(ignore)) { - LOG.warn("Skipping non-family directory" + pathName); - isIgnore = true; - break; - } - } - if (isIgnore) { - continue; - } - Path familyDir = stat.getPath(); - LOG.debug("Parsing family dir [" + familyDir.toString() + " in region [" + regionDir + "]"); - // Skip _logs, etc - if (familyDir.getName().startsWith("_") || familyDir.getName().startsWith(".")) { - continue; - } - - // start to parse hfile inside one family dir - Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); - for (Path hfile : hfiles) { - if (hfile.getName().startsWith("_") || hfile.getName().startsWith(".") - || StoreFileInfo.isReference(hfile.getName()) - || HFileLink.isHFileLink(hfile.getName())) { - continue; - } - HFile.Reader reader = HFile.createReader(fs, hfile, conf); - final byte[] first, last; - try { - reader.loadFileInfo(); - first = reader.getFirstRowKey().get(); - last = reader.getLastRowKey().get(); - LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first=" - + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last)); - - // To eventually infer start key-end key boundaries - Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0; - map.put(first, value + 1); - value = map.containsKey(last) ? (Integer) map.get(last) : 0; - map.put(last, value - 1); - } finally { - reader.close(); - } - } - } - } - return LoadIncrementalHFiles.inferBoundaries(map); - } - - /** - * Prepare the table for bulkload, most codes copied from - * {@link LoadIncrementalHFiles#createTable(TableName, String, Admin)} - * @param conn connection - * @param tableBackupPath path - * @param tableName table name - * @param targetTableName target table name - * @param regionDirList region directory list - * @param htd table descriptor - * @param truncateIfExists truncates table if exists - * @throws IOException exception - */ - private void checkAndCreateTable(Connection conn, Path tableBackupPath, TableName tableName, - TableName targetTableName, ArrayList regionDirList, TableDescriptor htd, - boolean truncateIfExists) throws IOException { - try (Admin admin = conn.getAdmin();) { - boolean createNew = false; - if (admin.tableExists(targetTableName)) { - if (truncateIfExists) { - LOG.info("Truncating exising target table '" + targetTableName - + "', preserving region splits"); - admin.disableTable(targetTableName); - admin.truncateTable(targetTableName, true); - } else { - LOG.info("Using exising target table '" + targetTableName + "'"); - } - } else { - createNew = true; - } - if (createNew) { - LOG.info("Creating target table '" + targetTableName + "'"); - byte[][] keys = null; - if (regionDirList == null || regionDirList.size() == 0) { - admin.createTable(htd, null); - } else { - keys = generateBoundaryKeys(regionDirList); - // create table using table descriptor and region boundaries - admin.createTable(htd, keys); - } - - } - long startTime = EnvironmentEdgeManager.currentTime(); - while (!admin.isTableAvailable(targetTableName)) { - try { - Thread.sleep(100); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - if (EnvironmentEdgeManager.currentTime() - startTime > TABLE_AVAILABILITY_WAIT_TIME) { - throw new IOException("Time out " + TABLE_AVAILABILITY_WAIT_TIME + "ms expired, table " - + targetTableName + " is still not available"); - } - } - } - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java deleted file mode 100644 index c44efbdfeef..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ /dev/null @@ -1,508 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.impl.FullTableBackupClient; -import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager; -import org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Durability; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.wal.WALFactory; -import org.junit.AfterClass; -import org.junit.Before; - -/** - * This class is only a base for other integration-level backup tests. Do not add tests here. - * TestBackupSmallTests is where tests that don't require bring machines up/down should go All other - * tests should have their own classes and extend this one - */ -public class TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupBase.class); - - protected static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - protected static HBaseTestingUtility TEST_UTIL2; - protected static Configuration conf1 = TEST_UTIL.getConfiguration(); - protected static Configuration conf2; - - protected static TableName table1 = TableName.valueOf("table1"); - protected static HTableDescriptor table1Desc; - protected static TableName table2 = TableName.valueOf("table2"); - protected static TableName table3 = TableName.valueOf("table3"); - protected static TableName table4 = TableName.valueOf("table4"); - - protected static TableName table1_restore = TableName.valueOf("ns1:table1_restore"); - protected static TableName table2_restore = TableName.valueOf("ns2:table2_restore"); - protected static TableName table3_restore = TableName.valueOf("ns3:table3_restore"); - protected static TableName table4_restore = TableName.valueOf("ns4:table4_restore"); - - protected static final int NB_ROWS_IN_BATCH = 99; - protected static final byte[] qualName = Bytes.toBytes("q1"); - protected static final byte[] famName = Bytes.toBytes("f"); - - protected static String BACKUP_ROOT_DIR = Path.SEPARATOR +"backupUT"; - protected static String BACKUP_REMOTE_ROOT_DIR = Path.SEPARATOR + "backupUT"; - protected static String provider = "defaultProvider"; - protected static boolean secure = false; - - protected static boolean autoRestoreOnFailure = true; - protected static boolean setupIsDone = false; - protected static boolean useSecondCluster = false; - - - static class IncrementalTableBackupClientForTest extends IncrementalTableBackupClient - { - - public IncrementalTableBackupClientForTest() { - } - - public IncrementalTableBackupClientForTest(Connection conn, - String backupId, BackupRequest request) throws IOException { - super(conn, backupId, request); - } - - @Override - public void execute() throws IOException - { - // case INCREMENTAL_COPY: - try { - // case PREPARE_INCREMENTAL: - failStageIf(Stage.stage_0); - beginBackup(backupManager, backupInfo); - - failStageIf(Stage.stage_1); - backupInfo.setPhase(BackupPhase.PREPARE_INCREMENTAL); - LOG.debug("For incremental backup, current table set is " - + backupManager.getIncrementalBackupTableSet()); - newTimestamps = ((IncrementalBackupManager) backupManager).getIncrBackupLogFileMap(); - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); - // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT - convertWALsToHFiles(); - incrementalCopyHFiles(new String[] {getBulkOutputDir().toString()}, - backupInfo.getBackupRootDir()); - failStageIf(Stage.stage_2); - // Save list of WAL files copied - backupManager.recordWALFiles(backupInfo.getIncrBackupFileList()); - - // case INCR_BACKUP_COMPLETE: - // set overall backup status: complete. Here we make sure to complete the backup. - // After this checkpoint, even if entering cancel process, will let the backup finished - // Set the previousTimestampMap which is before this current log roll to the manifest. - HashMap> previousTimestampMap = - backupManager.readLogTimestampMap(); - backupInfo.setIncrTimestampMap(previousTimestampMap); - - // The table list in backupInfo is good for both full backup and incremental backup. - // For incremental backup, it contains the incremental backup table set. - backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); - failStageIf(Stage.stage_3); - - HashMap> newTableSetTimestampMap = - backupManager.readLogTimestampMap(); - - Long newStartCode = - BackupUtils.getMinValue(BackupUtils.getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); - - handleBulkLoad(backupInfo.getTableNames()); - failStageIf(Stage.stage_4); - - // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.INCREMENTAL, conf); - - } catch (Exception e) { - failBackup(conn, backupInfo, backupManager, e, "Unexpected Exception : ", - BackupType.INCREMENTAL, conf); - throw new IOException(e); - } - - } - } - - static class FullTableBackupClientForTest extends FullTableBackupClient - { - - - public FullTableBackupClientForTest() { - } - - public FullTableBackupClientForTest(Connection conn, String backupId, BackupRequest request) - throws IOException { - super(conn, backupId, request); - } - - @Override - public void execute() throws IOException - { - // Get the stage ID to fail on - try (Admin admin = conn.getAdmin();) { - // Begin BACKUP - beginBackup(backupManager, backupInfo); - failStageIf(Stage.stage_0); - String savedStartCode = null; - boolean firstBackup = false; - // do snapshot for full table backup - savedStartCode = backupManager.readBackupStartCode(); - firstBackup = savedStartCode == null || Long.parseLong(savedStartCode) == 0L; - if (firstBackup) { - // This is our first backup. Let's put some marker to system table so that we can hold the logs - // while we do the backup. - backupManager.writeBackupStartCode(0L); - } - failStageIf(Stage.stage_1); - // We roll log here before we do the snapshot. It is possible there is duplicate data - // in the log that is already in the snapshot. But if we do it after the snapshot, we - // could have data loss. - // A better approach is to do the roll log on each RS in the same global procedure as - // the snapshot. - LOG.info("Execute roll log procedure for full backup ..."); - - Map props = new HashMap(); - props.put("backupRoot", backupInfo.getBackupRootDir()); - admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, - LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); - failStageIf(Stage.stage_2); - newTimestamps = backupManager.readRegionServerLastLogRollResult(); - if (firstBackup) { - // Updates registered log files - // We record ALL old WAL files as registered, because - // this is a first full backup in the system and these - // files are not needed for next incremental backup - List logFiles = BackupUtils.getWALFilesOlderThan(conf, newTimestamps); - backupManager.recordWALFiles(logFiles); - } - - // SNAPSHOT_TABLES: - backupInfo.setPhase(BackupPhase.SNAPSHOT); - for (TableName tableName : tableList) { - String snapshotName = - "snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime()) + "_" - + tableName.getNamespaceAsString() + "_" + tableName.getQualifierAsString(); - - snapshotTable(admin, tableName, snapshotName); - backupInfo.setSnapshotName(tableName, snapshotName); - } - failStageIf(Stage.stage_3); - // SNAPSHOT_COPY: - // do snapshot copy - LOG.debug("snapshot copy for " + backupId); - snapshotCopy(backupInfo); - // Updates incremental backup table set - backupManager.addIncrementalBackupTableSet(backupInfo.getTables()); - - // BACKUP_COMPLETE: - // set overall backup status: complete. Here we make sure to complete the backup. - // After this checkpoint, even if entering cancel process, will let the backup finished - backupInfo.setState(BackupState.COMPLETE); - // The table list in backupInfo is good for both full backup and incremental backup. - // For incremental backup, it contains the incremental backup table set. - backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); - - HashMap> newTableSetTimestampMap = - backupManager.readLogTimestampMap(); - - Long newStartCode = - BackupUtils.getMinValue(BackupUtils - .getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); - failStageIf(Stage.stage_4); - // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.FULL, conf); - - } catch (Exception e) { - - if(autoRestoreOnFailure) { - failBackup(conn, backupInfo, backupManager, e, "Unexpected BackupException : ", - BackupType.FULL, conf); - } - throw new IOException(e); - } - } - - } - - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - if (setupIsDone) { - return; - } - if (secure) { - // set the always on security provider - UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - // setup configuration - SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); - } - String coproc = conf1.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); - conf1.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, (coproc == null ? "" : coproc + ",") + - BackupObserver.class.getName()); - conf1.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); - BackupManager.decorateMasterConfiguration(conf1); - BackupManager.decorateRegionServerConfiguration(conf1); - conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); - // Set MultiWAL (with 2 default WAL files per RS) - conf1.set(WALFactory.WAL_PROVIDER, provider); - TEST_UTIL.startMiniCluster(); - - if (useSecondCluster) { - conf2 = HBaseConfiguration.create(conf1); - conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); - TEST_UTIL2 = new HBaseTestingUtility(conf2); - TEST_UTIL2.setZkCluster(TEST_UTIL.getZkCluster()); - TEST_UTIL2.startMiniCluster(); - } - conf1 = TEST_UTIL.getConfiguration(); - - TEST_UTIL.startMiniMapReduceCluster(); - BACKUP_ROOT_DIR = - new Path ( new Path(TEST_UTIL.getConfiguration().get("fs.defaultFS")), - BACKUP_ROOT_DIR).toString(); - LOG.info("ROOTDIR " + BACKUP_ROOT_DIR); - if (useSecondCluster) { - BACKUP_REMOTE_ROOT_DIR = - new Path ( new Path(TEST_UTIL2.getConfiguration().get("fs.defaultFS")) - + BACKUP_REMOTE_ROOT_DIR).toString(); - LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR); - } - createTables(); - populateFromMasterConfig(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), conf1); - setupIsDone = true; - } - - private static void populateFromMasterConfig(Configuration masterConf, Configuration conf) { - Iterator> it = masterConf.iterator(); - while (it.hasNext()) { - Entry e = it.next(); - conf.set(e.getKey(), e.getValue()); - } - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDown() throws Exception { - try{ - SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin()); - } catch (Exception e) { - } - SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); - if (useSecondCluster) { - TEST_UTIL2.shutdownMiniCluster(); - } - TEST_UTIL.shutdownMiniCluster(); - TEST_UTIL.shutdownMiniMapReduceCluster(); - } - - HTable insertIntoTable(Connection conn, TableName table, byte[] family, int id, int numRows) - throws IOException { - HTable t = (HTable) conn.getTable(table); - Put p1; - for (int i = 0; i < numRows; i++) { - p1 = new Put(Bytes.toBytes("row-" + table + "-" + id + "-" + i)); - p1.addColumn(family, qualName, Bytes.toBytes("val" + i)); - t.put(p1); - } - return t; - } - - - protected BackupRequest createBackupRequest(BackupType type, - List tables, String path) { - BackupRequest.Builder builder = new BackupRequest.Builder(); - BackupRequest request = builder.withBackupType(type) - .withTableList(tables) - .withTargetRootDir(path).build(); - return request; - } - - protected String backupTables(BackupType type, List tables, String path) - throws IOException { - Connection conn = null; - BackupAdmin badmin = null; - String backupId; - try { - conn = ConnectionFactory.createConnection(conf1); - badmin = new BackupAdminImpl(conn); - BackupRequest request = createBackupRequest(type, tables, path); - backupId = badmin.backupTables(request); - } finally { - if (badmin != null) { - badmin.close(); - } - if (conn != null) { - conn.close(); - } - } - return backupId; - } - - protected String fullTableBackup(List tables) throws IOException { - return backupTables(BackupType.FULL, tables, BACKUP_ROOT_DIR); - } - - protected String incrementalTableBackup(List tables) throws IOException { - return backupTables(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - } - - protected static void loadTable(Table table) throws Exception { - - Put p; // 100 + 1 row to t1_syncup - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p = new Put(Bytes.toBytes("row" + i)); - p.setDurability(Durability.SKIP_WAL); - p.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - table.put(p); - } - } - - protected static void createTables() throws Exception { - - long tid = System.currentTimeMillis(); - table1 = TableName.valueOf("ns1:test-" + tid); - HBaseAdmin ha = TEST_UTIL.getHBaseAdmin(); - - // Create namespaces - NamespaceDescriptor desc1 = NamespaceDescriptor.create("ns1").build(); - NamespaceDescriptor desc2 = NamespaceDescriptor.create("ns2").build(); - NamespaceDescriptor desc3 = NamespaceDescriptor.create("ns3").build(); - NamespaceDescriptor desc4 = NamespaceDescriptor.create("ns4").build(); - - ha.createNamespace(desc1); - ha.createNamespace(desc2); - ha.createNamespace(desc3); - ha.createNamespace(desc4); - - HTableDescriptor desc = new HTableDescriptor(table1); - HColumnDescriptor fam = new HColumnDescriptor(famName); - desc.addFamily(fam); - ha.createTable(desc); - table1Desc = desc; - Connection conn = ConnectionFactory.createConnection(conf1); - Table table = conn.getTable(table1); - loadTable(table); - table.close(); - table2 = TableName.valueOf("ns2:test-" + tid + 1); - desc = new HTableDescriptor(table2); - desc.addFamily(fam); - ha.createTable(desc); - table = conn.getTable(table2); - loadTable(table); - table.close(); - table3 = TableName.valueOf("ns3:test-" + tid + 2); - table = TEST_UTIL.createTable(table3, famName); - table.close(); - table4 = TableName.valueOf("ns4:test-" + tid + 3); - table = TEST_UTIL.createTable(table4, famName); - table.close(); - ha.close(); - conn.close(); - } - - protected boolean checkSucceeded(String backupId) throws IOException { - BackupInfo status = getBackupInfo(backupId); - if (status == null) return false; - return status.getState() == BackupState.COMPLETE; - } - - protected boolean checkFailed(String backupId) throws IOException { - BackupInfo status = getBackupInfo(backupId); - if (status == null) return false; - return status.getState() == BackupState.FAILED; - } - - private BackupInfo getBackupInfo(String backupId) throws IOException { - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - BackupInfo status = table.readBackupInfo(backupId); - return status; - } - } - - protected BackupAdmin getBackupAdmin() throws IOException { - return new BackupAdminImpl(TEST_UTIL.getConnection()); - } - - /** - * Helper method - */ - protected List toList(String... args) { - List ret = new ArrayList<>(); - for (int i = 0; i < args.length; i++) { - ret.add(TableName.valueOf(args[i])); - } - return ret; - } - - protected void dumpBackupDir() throws IOException { - // Dump Backup Dir - FileSystem fs = FileSystem.get(conf1); - RemoteIterator it = fs.listFiles(new Path(BACKUP_ROOT_DIR), true); - while (it.hasNext()) { - LOG.debug(it.next().getPath()); - } - - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java deleted file mode 100644 index 4670c49013d..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestBackupBoundaryTests extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupBoundaryTests.class); - - /** - * Verify that full backup is created on a single empty table correctly. - * @throws Exception - */ - @Test - public void testFullBackupSingleEmpty() throws Exception { - - LOG.info("create full backup image on single table"); - List tables = Lists.newArrayList(table3); - LOG.info("Finished Backup " + fullTableBackup(tables)); - } - - /** - * Verify that full backup is created on multiple empty tables correctly. - * @throws Exception - */ - @Test - public void testFullBackupMultipleEmpty() throws Exception { - LOG.info("create full backup image on mulitple empty tables"); - - List tables = Lists.newArrayList(table3, table4); - fullTableBackup(tables); - } - - /** - * Verify that full backup fails on a single table that does not exist. - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFullBackupSingleDNE() throws Exception { - - LOG.info("test full backup fails on a single table that does not exist"); - List tables = toList("tabledne"); - fullTableBackup(tables); - } - - /** - * Verify that full backup fails on multiple tables that do not exist. - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFullBackupMultipleDNE() throws Exception { - - LOG.info("test full backup fails on multiple tables that do not exist"); - List tables = toList("table1dne", "table2dne"); - fullTableBackup(tables); - } - - /** - * Verify that full backup fails on tableset containing real and fake tables. - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFullBackupMixExistAndDNE() throws Exception { - LOG.info("create full backup fails on tableset containing real and fake table"); - - List tables = toList(table1.getNameAsString(), "tabledne"); - fullTableBackup(tables); - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java deleted file mode 100644 index c9fc84ff66c..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java +++ /dev/null @@ -1,441 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(SmallTests.class) -public class TestBackupCommandLineTool { - - private final static String USAGE_DESCRIBE = "Usage: hbase backup describe "; - private final static String USAGE_CREATE = "Usage: hbase backup create"; - private final static String USAGE_HISTORY = "Usage: hbase backup history"; - private final static String USAGE_BACKUP = "Usage: hbase backup"; - private final static String USAGE_DELETE = "Usage: hbase backup delete"; - private final static String USAGE_PROGRESS = "Usage: hbase backup progress"; - private final static String USAGE_SET = "Usage: hbase backup set"; - private final static String USAGE_RESTORE = "Usage: hbase restore"; - - Configuration conf; - - @Before - public void setUpBefore() throws Exception { - conf = HBaseConfiguration.create(); - conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); - } - - @Test - public void testBackupDriverDescribeHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "describe", "-help" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_DESCRIBE) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "describe", "-h" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_DESCRIBE) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "describe" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_DESCRIBE) >= 0); - } - - - @Test - public void testBackupDriverCreateTopLevelBackupDest() throws Exception { - String[] args = new String[] { "create", "full", "hdfs://localhost:1020", "-t", "t1" }; - int result = ToolRunner.run(conf, new BackupDriver(), args); - // FAILED - assertEquals(1, result); - } - - @Test - public void testBackupDriverCreateHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "create", "-help" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_CREATE) >= 0); - assertTrue(output.indexOf(BackupRestoreConstants.OPTION_TABLE_LIST_DESC) > 0); - - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "create", "-h" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_CREATE) >= 0); - assertTrue(output.indexOf(BackupRestoreConstants.OPTION_TABLE_LIST_DESC) > 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "create" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_CREATE) >= 0); - assertTrue(output.indexOf(BackupRestoreConstants.OPTION_TABLE_LIST_DESC) > 0); - - } - - @Test - public void testBackupDriverHistoryHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "history", "-help" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_HISTORY) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "history", "-h" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_HISTORY) >= 0); - - } - - @Test - public void testBackupDriverDeleteHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "delete", "-help" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_DELETE) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "delete", "-h" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_DELETE) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "delete" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_DELETE) >= 0); - } - - @Test - public void testBackupDriverProgressHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "progress", "-help" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_PROGRESS) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "progress", "-h" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_PROGRESS) >= 0); - } - - @Test - public void testBackupDriverSetHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "set", "-help" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_SET) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "set", "-h" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_SET) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "set" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_SET) >= 0); - - } - - @Test - public void testBackupDriverHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "-help" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "-h" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - - } - - @Test - public void testRestoreDriverHelp() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "-help" }; - ToolRunner.run(conf, new RestoreDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_RESTORE) >= 0); - assertTrue(output.indexOf(BackupRestoreConstants.OPTION_TABLE_LIST_DESC) > 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "-h" }; - ToolRunner.run(conf, new RestoreDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_RESTORE) >= 0); - assertTrue(output.indexOf(BackupRestoreConstants.OPTION_TABLE_LIST_DESC) > 0); - - } - - @Test - public void testBackupDriverUnrecognizedCommand() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "command" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "command" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - } - - @Test - public void testBackupDriverUnrecognizedOption() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "create", "-xx" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "describe", "-xx" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "history", "-xx" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "delete", "-xx" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "set", "-xx" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_BACKUP) >= 0); - } - - @Test - public void testRestoreDriverUnrecognizedOption() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "-xx" }; - ToolRunner.run(conf, new RestoreDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_RESTORE) >= 0); - - } - - @Test - public void testBackupDriverCreateWrongArgNumber() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "create" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_CREATE) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "create", "22" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_CREATE) >= 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - args = new String[] { "create", "22", "22", "22", "22", "22" }; - ToolRunner.run(conf, new BackupDriver(), args); - - output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_CREATE) >= 0); - } - - @Test - public void testBackupDriverDeleteWrongArgNumber() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "delete" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_DELETE) >= 0); - - } - - @Test - public void testBackupDriverHistoryWrongArgs() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "history", "-n", "xx" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf(USAGE_HISTORY) >= 0); - - } - - @Test - public void testBackupDriverWrongBackupDestination() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "create", "full", "clicks" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf("ERROR: invalid backup destination") >= 0); - - } - - @Test - public void testBackupDriverBackupSetAndList() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] args = new String[] { "create", "full", "file:/localhost", "-t", "clicks", "-s", "s" }; - ToolRunner.run(conf, new BackupDriver(), args); - - String output = baos.toString(); - System.out.println(baos.toString()); - assertTrue(output.indexOf("ERROR: You can specify either backup set or list") >= 0); - - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java deleted file mode 100644 index ebfc7355139..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestBackupDelete extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupDelete.class); - - /** - * Verify that full backup is created on a single table with data correctly. Verify that history - * works as expected - * @throws Exception - */ - @Test - public void testBackupDelete() throws Exception { - LOG.info("test backup delete on a single table with data"); - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId)); - LOG.info("backup complete"); - String[] backupIds = new String[] { backupId }; - BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection()); - BackupInfo info = table.readBackupInfo(backupId); - Path path = new Path(info.getBackupRootDir(), backupId); - FileSystem fs = FileSystem.get(path.toUri(), conf1); - assertTrue(fs.exists(path)); - int deleted = getBackupAdmin().deleteBackups(backupIds); - - assertTrue(!fs.exists(path)); - assertTrue(fs.exists(new Path(info.getBackupRootDir()))); - assertTrue(1 == deleted); - table.close(); - LOG.info("delete_backup"); - } - - /** - * Verify that full backup is created on a single table with data correctly. Verify that history - * works as expected - * @throws Exception - */ - @Test - public void testBackupDeleteCommand() throws Exception { - LOG.info("test backup delete on a single table with data: command-line"); - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId)); - LOG.info("backup complete"); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - - String[] args = new String[] { "delete", backupId }; - // Run backup - - try { - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - } catch (Exception e) { - LOG.error("failed", e); - } - LOG.info("delete_backup"); - String output = baos.toString(); - LOG.info(baos.toString()); - assertTrue(output.indexOf("Deleted 1 backups") >= 0); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java deleted file mode 100644 index 85fba4b5aef..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(MediumTests.class) -public class TestBackupDeleteRestore extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupDeleteRestore.class); - - /** - * Verify that load data- backup - delete some data - restore works as expected - deleted data get - * restored. - * @throws Exception - */ - @Test - public void testBackupDeleteRestore() throws Exception { - - LOG.info("test full restore on a single table empty table"); - - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - LOG.info("backup complete"); - int numRows = TEST_UTIL.countRows(table1); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - // delete row - try (Table table = TEST_UTIL.getConnection().getTable(table1);) { - Delete delete = new Delete("row0".getBytes()); - table.delete(delete); - hba.flush(table1); - } - - TableName[] tableset = new TableName[] { table1 }; - TableName[] tablemap = null;// new TableName[] { table1_restore }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - tableset, tablemap, true)); - - int numRowsAfterRestore = TEST_UTIL.countRows(table1); - assertEquals(numRows, numRowsAfterRestore); - hba.close(); - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java deleted file mode 100644 index af8e90785f7..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.MasterObserver; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -/** - * This class is only a base for other integration-level backup tests. Do not add tests here. - * TestBackupSmallTests is where tests that don't require bring machines up/down should go All other - * tests should have their own classes and extend this one - */ -@Category(LargeTests.class) -public class TestBackupDeleteWithFailures extends TestBackupBase{ - - private static final Log LOG = LogFactory.getLog(TestBackupDeleteWithFailures.class); - - - - public static enum Failure { - NO_FAILURES, - PRE_SNAPSHOT_FAILURE, - PRE_DELETE_SNAPSHOT_FAILURE, - POST_DELETE_SNAPSHOT_FAILURE - } - - public static class MasterSnapshotObserver implements MasterCoprocessor, MasterObserver { - List failures = new ArrayList(); - - public void setFailures(Failure ... f) { - failures.clear(); - for (int i = 0; i < f.length; i++) { - failures.add(f[i]); - } - } - - @Override - public Optional getMasterObserver() { - return Optional.of(this); - } - - @Override - public void preSnapshot(final ObserverContext ctx, - final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor) - throws IOException - { - if (failures.contains(Failure.PRE_SNAPSHOT_FAILURE)) { - throw new IOException ("preSnapshot"); - } - } - - @Override - public void preDeleteSnapshot(ObserverContext ctx, - SnapshotDescription snapshot) throws IOException { - if (failures.contains(Failure.PRE_DELETE_SNAPSHOT_FAILURE)) { - throw new IOException ("preDeleteSnapshot"); - } - } - - @Override - public void postDeleteSnapshot(ObserverContext ctx, - SnapshotDescription snapshot) throws IOException { - if (failures.contains(Failure.POST_DELETE_SNAPSHOT_FAILURE)) { - throw new IOException ("postDeleteSnapshot"); - } - } - - } - - /** - * @throws java.lang.Exception - */ - @Override - @Before - public void setUp() throws Exception { - conf1.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, - MasterSnapshotObserver.class.getName()); - conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); - super.setUp(); - } - - - private MasterSnapshotObserver getMasterSnapshotObserver() { - return TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost() - .findCoprocessor(MasterSnapshotObserver.class); - } - - @Test - public void testBackupDeleteWithFailures() throws Exception - { - testBackupDeleteWithFailuresAfter(1, Failure.PRE_DELETE_SNAPSHOT_FAILURE); - testBackupDeleteWithFailuresAfter(0, Failure.POST_DELETE_SNAPSHOT_FAILURE); - testBackupDeleteWithFailuresAfter(1, Failure.PRE_SNAPSHOT_FAILURE); - } - - private void testBackupDeleteWithFailuresAfter(int expected, Failure ...failures) throws Exception { - LOG.info("test repair backup delete on a single table with data and failures "+ failures[0]); - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId)); - LOG.info("backup complete"); - String[] backupIds = new String[] { backupId }; - BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection()); - BackupInfo info = table.readBackupInfo(backupId); - Path path = new Path(info.getBackupRootDir(), backupId); - FileSystem fs = FileSystem.get(path.toUri(), conf1); - assertTrue(fs.exists(path)); - - Connection conn = TEST_UTIL.getConnection(); - Admin admin = conn.getAdmin(); - MasterSnapshotObserver observer = getMasterSnapshotObserver(); - - observer.setFailures(failures); - try { - getBackupAdmin().deleteBackups(backupIds); - } catch(IOException e) { - if(expected != 1) assertTrue(false); - } - - // Verify that history length == expected after delete failure - assertTrue (table.getBackupHistory().size() == expected); - - String[] ids = table.getListOfBackupIdsFromDeleteOperation(); - - // Verify that we still have delete record in backup system table - if(expected == 1) { - assertTrue(ids.length == 1); - assertTrue(ids[0].equals(backupId)); - } else { - assertNull(ids); - } - - // Now run repair command to repair "failed" delete operation - String[] args = new String[] {"repair"}; - - observer.setFailures(Failure.NO_FAILURES); - - // Run repair - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - // Verify that history length == 0 - assertTrue (table.getBackupHistory().size() == 0); - ids = table.getListOfBackupIdsFromDeleteOperation(); - - // Verify that we do not have delete record in backup system table - assertNull(ids); - - table.close(); - admin.close(); - } - - - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java deleted file mode 100644 index 06723258653..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.impl.BackupCommands; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestBackupDescribe extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupDescribe.class); - - /** - * Verify that describe works as expected if incorrect backup Id is supplied - * @throws Exception - */ - @Test - public void testBackupDescribe() throws Exception { - - LOG.info("test backup describe on a single table with data"); - - String[] args = new String[] { "describe", "backup_2" }; - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret < 0); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setErr(new PrintStream(baos)); - args = new String[] { "progress" }; - ToolRunner.run(TEST_UTIL.getConfiguration(), new BackupDriver(), args); - - String output = baos.toString(); - LOG.info("Output from progress: " + output); - assertTrue(output.indexOf(BackupCommands.NO_ACTIVE_SESSION_FOUND) >= 0); - } - - @Test - public void testBackupSetCommandWithNonExistentTable() throws Exception { - String[] args = new String[] { "set", "add", "some_set", "table" }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertNotEquals(ret, 0); - } - - @Test - public void testBackupDescribeCommand() throws Exception { - - LOG.info("test backup describe on a single table with data: command-line"); - - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - - BackupInfo info = getBackupAdmin().getBackupInfo(backupId); - assertTrue(info.getState() == BackupState.COMPLETE); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - - String[] args = new String[] { "describe", backupId }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - String response = baos.toString(); - assertTrue(response.indexOf(backupId) > 0); - assertTrue(response.indexOf("COMPLETE") > 0); - - BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection()); - BackupInfo status = table.readBackupInfo(backupId); - String desc = status.getShortDescription(); - table.close(); - assertTrue(response.indexOf(desc) >= 0); - - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java deleted file mode 100644 index 6c1890b4a8c..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({ MasterTests.class, SmallTests.class }) -public class TestBackupHFileCleaner { - private static final Log LOG = LogFactory.getLog(TestBackupHFileCleaner.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static Configuration conf = TEST_UTIL.getConfiguration(); - private static TableName tableName = TableName.valueOf("backup.hfile.cleaner"); - private static String famName = "fam"; - static FileSystem fs = null; - Path root; - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); - TEST_UTIL.startMiniZKCluster(); - TEST_UTIL.startMiniCluster(1); - fs = FileSystem.get(conf); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - if (fs != null) { - fs.close(); - } - TEST_UTIL.shutdownMiniCluster(); - } - - @Before - public void setup() throws IOException { - root = TEST_UTIL.getDataTestDirOnTestFS(); - } - - @After - public void cleanup() { - try { - fs.delete(root, true); - } catch (IOException e) { - LOG.warn("Failed to delete files recursively from path " + root); - } - } - - @Test - public void testGetDeletableFiles() throws IOException { - // 1. Create a file - Path file = new Path(root, "testIsFileDeletableWithNoHFileRefs"); - fs.createNewFile(file); - // 2. Assert file is successfully created - assertTrue("Test file not created!", fs.exists(file)); - BackupHFileCleaner cleaner = new BackupHFileCleaner(); - cleaner.setConf(conf); - cleaner.setCheckForFullyBackedUpTables(false); - // 3. Assert that file as is should be deletable - List stats = new ArrayList<>(); - FileStatus stat = fs.getFileStatus(file); - stats.add(stat); - Iterable deletable = cleaner.getDeletableFiles(stats); - deletable = cleaner.getDeletableFiles(stats); - boolean found = false; - for (FileStatus stat1 : deletable) { - if (stat.equals(stat1)) found = true; - } - assertTrue("Cleaner should allow to delete this file as there is no hfile reference " - + "for it.", found); - - // 4. Add the file as bulk load - List list = new ArrayList<>(1); - list.add(file); - try (Connection conn = ConnectionFactory.createConnection(conf); - BackupSystemTable sysTbl = new BackupSystemTable(conn)) { - List sTableList = new ArrayList<>(); - sTableList.add(tableName); - Map>[] maps = new Map[1]; - maps[0] = new HashMap<>(); - maps[0].put(famName.getBytes(), list); - sysTbl.writeBulkLoadedFiles(sTableList, maps, "1"); - } - - // 5. Assert file should not be deletable - deletable = cleaner.getDeletableFiles(stats); - deletable = cleaner.getDeletableFiles(stats); - found = false; - for (FileStatus stat1 : deletable) { - if (stat.equals(stat1)) found = true; - } - assertFalse("Cleaner should not allow to delete this file as there is a hfile reference " - + "for it.", found); - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java deleted file mode 100644 index 0944ea2cc7c..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -/** - * Create multiple backups for two tables: table1, table2 then perform 1 delete - */ -@Category(LargeTests.class) -public class TestBackupMultipleDeletes extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestBackupMultipleDeletes.class); - - @Test - public void testBackupMultipleDeletes() throws Exception { - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - List tables = Lists.newArrayList(table1, table2); - HBaseAdmin admin = null; - Connection conn = ConnectionFactory.createConnection(conf1); - admin = (HBaseAdmin) conn.getAdmin(); - BackupAdmin client = new BackupAdminImpl(conn); - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); - assertTrue(checkSucceeded(backupIdFull)); - // #2 - insert some data to table table1 - HTable t1 = (HTable) conn.getTable(table1); - Put p1; - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH * 2); - t1.close(); - // #3 - incremental backup for table1 - tables = Lists.newArrayList(table1); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc1 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdInc1)); - // #4 - insert some data to table table2 - HTable t2 = (HTable) conn.getTable(table2); - Put p2 = null; - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p2 = new Put(Bytes.toBytes("row-t2" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - // #5 - incremental backup for table1, table2 - tables = Lists.newArrayList(table1, table2); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc2 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdInc2)); - // #6 - insert some data to table table1 - t1 = (HTable) conn.getTable(table1); - for (int i = NB_ROWS_IN_BATCH; i < 2 * NB_ROWS_IN_BATCH; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - // #7 - incremental backup for table1 - tables = Lists.newArrayList(table1); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc3 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdInc3)); - // #8 - insert some data to table table2 - t2 = (HTable) conn.getTable(table2); - for (int i = NB_ROWS_IN_BATCH; i < 2 * NB_ROWS_IN_BATCH; i++) { - p2 = new Put(Bytes.toBytes("row-t1" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - // #9 - incremental backup for table1, table2 - tables = Lists.newArrayList(table1, table2); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc4 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdInc4)); - // #10 full backup for table3 - tables = Lists.newArrayList(table3); - request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull2 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdFull2)); - // #11 - incremental backup for table3 - tables = Lists.newArrayList(table3); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc5 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdInc5)); - LOG.error("Delete backupIdInc2"); - client.deleteBackups(new String[] { backupIdInc2 }); - LOG.error("Delete backupIdInc2 done"); - List list = client.getHistory(100); - // First check number of backup images before and after - assertEquals(4, list.size()); - // then verify that no backupIdInc2,3,4 - Set ids = new HashSet(); - ids.add(backupIdInc2); - ids.add(backupIdInc3); - ids.add(backupIdInc4); - for (BackupInfo info : list) { - String backupId = info.getBackupId(); - if (ids.contains(backupId)) { - assertTrue(false); - } - } - // Verify that backupInc5 contains only table3 - boolean found = false; - for (BackupInfo info : list) { - String backupId = info.getBackupId(); - if (backupId.equals(backupIdInc5)) { - assertTrue(info.getTables().size() == 1); - assertEquals(table3, info.getTableNames().get(0)); - found = true; - } - } - assertTrue(found); - admin.close(); - conn.close(); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java deleted file mode 100644 index 4b1d84f0869..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.List; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.impl.TableBackupClient; -import org.apache.hadoop.hbase.backup.impl.TableBackupClient.Stage; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - - -@Category(LargeTests.class) -public class TestBackupRepair extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupRepair.class); - - - @Test - public void testFullBackupWithFailuresAndRestore() throws Exception { - - autoRestoreOnFailure = false; - - conf1.set(TableBackupClient.BACKUP_CLIENT_IMPL_CLASS, - FullTableBackupClientForTest.class.getName()); - int maxStage = Stage.values().length -1; - // Fail stage in loop between 0 and 4 inclusive - for (int stage = 0; stage < maxStage; stage++) { - LOG.info("Running stage " + stage); - runBackupAndFailAtStageWithRestore(stage); - } - } - - public void runBackupAndFailAtStageWithRestore(int stage) throws Exception { - - conf1.setInt(FullTableBackupClientForTest.BACKUP_TEST_MODE_STAGE, stage); - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - int before = table.getBackupHistory().size(); - String[] args = - new String[] { "create", "full", BACKUP_ROOT_DIR, "-t", - table1.getNameAsString() + "," + table2.getNameAsString() }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertFalse(ret == 0); - - // Now run restore - args = new String[] {"repair"}; - - ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - - List backups = table.getBackupHistory(); - int after = table.getBackupHistory().size(); - - assertTrue(after == before +1); - for (BackupInfo data : backups) { - String backupId = data.getBackupId(); - assertFalse(checkSucceeded(backupId)); - } - Set tables = table.getIncrementalBackupTableSet(BACKUP_ROOT_DIR); - assertTrue(tables.size() == 0); - } - } - - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java deleted file mode 100644 index 57c15330119..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestBackupShowHistory extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupShowHistory.class); - - private boolean findBackup(List history, String backupId) { - assertTrue(history.size() > 0); - boolean success = false; - for (BackupInfo info : history) { - if (info.getBackupId().equals(backupId)) { - success = true; - break; - } - } - return success; - } - - /** - * Verify that full backup is created on a single table with data correctly. Verify that history - * works as expected - * @throws Exception - */ - @Test - public void testBackupHistory() throws Exception { - - LOG.info("test backup history on a single table with data"); - - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId)); - LOG.info("backup complete"); - - List history = getBackupAdmin().getHistory(10); - assertTrue(findBackup(history, backupId)); - BackupInfo.Filter nullFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - return true; - } - }; - history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), nullFilter); - assertTrue(findBackup(history, backupId)); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - - String[] args = new String[] { "history", "-n", "10", "-p", BACKUP_ROOT_DIR }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - LOG.info("show_history"); - String output = baos.toString(); - LOG.info(output); - baos.close(); - assertTrue(output.indexOf(backupId) > 0); - - tableList = Lists.newArrayList(table2); - String backupId2 = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId2)); - LOG.info("backup complete: " + table2); - BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo image) { - if (table1 == null) return true; - List names = image.getTableNames(); - return names.contains(table1); - } - }; - BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - String backupId = info.getBackupId(); - return backupId.startsWith("backup"); - } - }; - - history = getBackupAdmin().getHistory(10, tableNameFilter, tableSetFilter); - assertTrue(history.size() > 0); - boolean success = true; - for (BackupInfo info : history) { - if (!info.getTableNames().contains(table1)) { - success = false; - break; - } - } - assertTrue(success); - - history = - BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), tableNameFilter, - tableSetFilter); - assertTrue(history.size() > 0); - success = true; - for (BackupInfo info : history) { - if (!info.getTableNames().contains(table1)) { - success = false; - break; - } - } - assertTrue(success); - - args = - new String[] { "history", "-n", "10", "-p", BACKUP_ROOT_DIR, - "-t", "table1", "-s", "backup" }; - // Run backup - ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - LOG.info("show_history"); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java deleted file mode 100644 index 73d8d9fd300..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestBackupStatusProgress extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupStatusProgress.class); - - /** - * Verify that full backup is created on a single table with data correctly. - * @throws Exception - */ - @Test - public void testBackupStatusProgress() throws Exception { - - LOG.info("test backup status/progress on a single table with data"); - - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - - BackupInfo info = getBackupAdmin().getBackupInfo(backupId); - assertTrue(info.getState() == BackupState.COMPLETE); - - LOG.debug(info.getShortDescription()); - assertTrue(info.getProgress() > 0); - - } - - @Test - public void testBackupStatusProgressCommand() throws Exception { - - LOG.info("test backup status/progress on a single table with data: command-line"); - - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - - String[] args = new String[] { "describe", backupId }; - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - String responce = baos.toString(); - assertTrue(responce.indexOf(backupId) > 0); - assertTrue(responce.indexOf("COMPLETE") > 0); - - baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - - args = new String[] { "progress", backupId }; - ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - responce = baos.toString(); - assertTrue(responce.indexOf(backupId) >= 0); - assertTrue(responce.indexOf("progress") > 0); - assertTrue(responce.indexOf("100") > 0); - - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java deleted file mode 100644 index 5814d87aa0d..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java +++ /dev/null @@ -1,511 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test cases for backup system table API - */ -@Category(MediumTests.class) -public class TestBackupSystemTable { - - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - protected static Configuration conf = UTIL.getConfiguration(); - protected static MiniHBaseCluster cluster; - protected static Connection conn; - protected BackupSystemTable table; - - @BeforeClass - public static void setUp() throws Exception { - conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); - BackupManager.decorateMasterConfiguration(conf); - BackupManager.decorateRegionServerConfiguration(conf); - cluster = UTIL.startMiniCluster(); - conn = UTIL.getConnection(); - } - - @Before - public void before() throws IOException { - table = new BackupSystemTable(conn); - } - - @After - public void after() { - if (table != null) { - table.close(); - } - - } - - @Test - public void testUpdateReadDeleteBackupStatus() throws IOException { - BackupInfo ctx = createBackupInfo(); - table.updateBackupInfo(ctx); - BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId()); - assertTrue(compare(ctx, readCtx)); - // try fake backup id - readCtx = table.readBackupInfo("fake"); - assertNull(readCtx); - // delete backup info - table.deleteBackupInfo(ctx.getBackupId()); - readCtx = table.readBackupInfo(ctx.getBackupId()); - assertNull(readCtx); - cleanBackupTable(); - } - - @Test - public void testWriteReadBackupStartCode() throws IOException { - Long code = 100L; - table.writeBackupStartCode(code, "root"); - String readCode = table.readBackupStartCode("root"); - assertEquals(code, new Long(Long.parseLong(readCode))); - cleanBackupTable(); - } - - private void cleanBackupTable() throws IOException { - Admin admin = UTIL.getHBaseAdmin(); - admin.disableTable(BackupSystemTable.getTableName(conf)); - admin.truncateTable(BackupSystemTable.getTableName(conf), true); - if (admin.isTableDisabled(BackupSystemTable.getTableName(conf))) { - admin.enableTable(BackupSystemTable.getTableName(conf)); - } - } - - @Test - public void testBackupHistory() throws IOException { - int n = 10; - List list = createBackupInfoList(n); - - // Load data - for (BackupInfo bc : list) { - // Make sure we set right status - bc.setState(BackupState.COMPLETE); - table.updateBackupInfo(bc); - } - - // Reverse list for comparison - Collections.reverse(list); - List history = table.getBackupHistory(); - assertTrue(history.size() == n); - - for (int i = 0; i < n; i++) { - BackupInfo ctx = list.get(i); - BackupInfo data = history.get(i); - assertTrue(compare(ctx, data)); - } - - cleanBackupTable(); - - } - - @Test - public void testBackupDelete() throws IOException { - - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - int n = 10; - List list = createBackupInfoList(n); - - // Load data - for (BackupInfo bc : list) { - // Make sure we set right status - bc.setState(BackupState.COMPLETE); - table.updateBackupInfo(bc); - } - - // Verify exists - for (BackupInfo bc : list) { - assertNotNull(table.readBackupInfo(bc.getBackupId())); - } - - // Delete all - for (BackupInfo bc : list) { - table.deleteBackupInfo(bc.getBackupId()); - } - - // Verify do not exists - for (BackupInfo bc : list) { - assertNull(table.readBackupInfo(bc.getBackupId())); - } - - cleanBackupTable(); - } - - } - - @Test - public void testRegionServerLastLogRollResults() throws IOException { - String[] servers = new String[] { "server1", "server2", "server3" }; - Long[] timestamps = new Long[] { 100L, 102L, 107L }; - - for (int i = 0; i < servers.length; i++) { - table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root"); - } - - HashMap result = table.readRegionServerLastLogRollResult("root"); - assertTrue(servers.length == result.size()); - Set keys = result.keySet(); - String[] keysAsArray = new String[keys.size()]; - keys.toArray(keysAsArray); - Arrays.sort(keysAsArray); - - for (int i = 0; i < keysAsArray.length; i++) { - assertEquals(keysAsArray[i], servers[i]); - Long ts1 = timestamps[i]; - Long ts2 = result.get(keysAsArray[i]); - assertEquals(ts1, ts2); - } - - cleanBackupTable(); - } - - @Test - public void testIncrementalBackupTableSet() throws IOException { - TreeSet tables1 = new TreeSet<>(); - - tables1.add(TableName.valueOf("t1")); - tables1.add(TableName.valueOf("t2")); - tables1.add(TableName.valueOf("t3")); - - TreeSet tables2 = new TreeSet<>(); - - tables2.add(TableName.valueOf("t3")); - tables2.add(TableName.valueOf("t4")); - tables2.add(TableName.valueOf("t5")); - - table.addIncrementalBackupTableSet(tables1, "root"); - BackupSystemTable table = new BackupSystemTable(conn); - TreeSet res1 = (TreeSet) table.getIncrementalBackupTableSet("root"); - assertTrue(tables1.size() == res1.size()); - Iterator desc1 = tables1.descendingIterator(); - Iterator desc2 = res1.descendingIterator(); - while (desc1.hasNext()) { - assertEquals(desc1.next(), desc2.next()); - } - - table.addIncrementalBackupTableSet(tables2, "root"); - TreeSet res2 = (TreeSet) table.getIncrementalBackupTableSet("root"); - assertTrue((tables2.size() + tables1.size() - 1) == res2.size()); - - tables1.addAll(tables2); - - desc1 = tables1.descendingIterator(); - desc2 = res2.descendingIterator(); - - while (desc1.hasNext()) { - assertEquals(desc1.next(), desc2.next()); - } - cleanBackupTable(); - - } - - @Test - public void testRegionServerLogTimestampMap() throws IOException { - TreeSet tables = new TreeSet<>(); - - tables.add(TableName.valueOf("t1")); - tables.add(TableName.valueOf("t2")); - tables.add(TableName.valueOf("t3")); - - HashMap rsTimestampMap = new HashMap(); - - rsTimestampMap.put("rs1:100", 100L); - rsTimestampMap.put("rs2:100", 101L); - rsTimestampMap.put("rs3:100", 103L); - - table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root"); - - HashMap> result = table.readLogTimestampMap("root"); - - assertTrue(tables.size() == result.size()); - - for (TableName t : tables) { - HashMap rstm = result.get(t); - assertNotNull(rstm); - assertEquals(rstm.get("rs1:100"), new Long(100L)); - assertEquals(rstm.get("rs2:100"), new Long(101L)); - assertEquals(rstm.get("rs3:100"), new Long(103L)); - } - - Set tables1 = new TreeSet<>(); - - tables1.add(TableName.valueOf("t3")); - tables1.add(TableName.valueOf("t4")); - tables1.add(TableName.valueOf("t5")); - - HashMap rsTimestampMap1 = new HashMap(); - - rsTimestampMap1.put("rs1:100", 200L); - rsTimestampMap1.put("rs2:100", 201L); - rsTimestampMap1.put("rs3:100", 203L); - - table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root"); - - result = table.readLogTimestampMap("root"); - - assertTrue(5 == result.size()); - - for (TableName t : tables) { - HashMap rstm = result.get(t); - assertNotNull(rstm); - if (t.equals(TableName.valueOf("t3")) == false) { - assertEquals(rstm.get("rs1:100"), new Long(100L)); - assertEquals(rstm.get("rs2:100"), new Long(101L)); - assertEquals(rstm.get("rs3:100"), new Long(103L)); - } else { - assertEquals(rstm.get("rs1:100"), new Long(200L)); - assertEquals(rstm.get("rs2:100"), new Long(201L)); - assertEquals(rstm.get("rs3:100"), new Long(203L)); - } - } - - for (TableName t : tables1) { - HashMap rstm = result.get(t); - assertNotNull(rstm); - assertEquals(rstm.get("rs1:100"), new Long(200L)); - assertEquals(rstm.get("rs2:100"), new Long(201L)); - assertEquals(rstm.get("rs3:100"), new Long(203L)); - } - - cleanBackupTable(); - - } - - @Test - public void testAddWALFiles() throws IOException { - List files = - Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1", - "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2", - "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3"); - String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5"; - - table.addWALFiles(files, "backup", "root"); - - assertTrue(table.isWALFileDeletable(files.get(0))); - assertTrue(table.isWALFileDeletable(files.get(1))); - assertTrue(table.isWALFileDeletable(files.get(2))); - assertFalse(table.isWALFileDeletable(newFile)); - - cleanBackupTable(); - } - - /** - * Backup set tests - */ - - @Test - public void testBackupSetAddNotExists() throws IOException { - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - String[] tables = new String[] { "table1", "table2", "table3" }; - String setName = "name"; - table.addToBackupSet(setName, tables); - List tnames = table.describeBackupSet(setName); - assertTrue(tnames != null); - assertTrue(tnames.size() == tables.length); - for (int i = 0; i < tnames.size(); i++) { - assertTrue(tnames.get(i).getNameAsString().equals(tables[i])); - } - cleanBackupTable(); - } - - } - - @Test - public void testBackupSetAddExists() throws IOException { - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - String[] tables = new String[] { "table1", "table2", "table3" }; - String setName = "name"; - table.addToBackupSet(setName, tables); - String[] addTables = new String[] { "table4", "table5", "table6" }; - table.addToBackupSet(setName, addTables); - - List tnames = table.describeBackupSet(setName); - assertTrue(tnames != null); - assertTrue(tnames.size() == tables.length + addTables.length); - for (int i = 0; i < tnames.size(); i++) { - assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1))); - } - cleanBackupTable(); - } - } - - @Test - public void testBackupSetAddExistsIntersects() throws IOException { - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - String[] tables = new String[] { "table1", "table2", "table3" }; - String setName = "name"; - table.addToBackupSet(setName, tables); - String[] addTables = new String[] { "table3", "table4", "table5", "table6" }; - table.addToBackupSet(setName, addTables); - - List tnames = table.describeBackupSet(setName); - assertTrue(tnames != null); - assertTrue(tnames.size() == tables.length + addTables.length - 1); - for (int i = 0; i < tnames.size(); i++) { - assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1))); - } - cleanBackupTable(); - } - } - - @Test - public void testBackupSetRemoveSomeNotExists() throws IOException { - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - String[] tables = new String[] { "table1", "table2", "table3", "table4" }; - String setName = "name"; - table.addToBackupSet(setName, tables); - String[] removeTables = new String[] { "table4", "table5", "table6" }; - table.removeFromBackupSet(setName, removeTables); - - List tnames = table.describeBackupSet(setName); - assertTrue(tnames != null); - assertTrue(tnames.size() == tables.length - 1); - for (int i = 0; i < tnames.size(); i++) { - assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1))); - } - cleanBackupTable(); - } - } - - @Test - public void testBackupSetRemove() throws IOException { - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - String[] tables = new String[] { "table1", "table2", "table3", "table4" }; - String setName = "name"; - table.addToBackupSet(setName, tables); - String[] removeTables = new String[] { "table4", "table3" }; - table.removeFromBackupSet(setName, removeTables); - - List tnames = table.describeBackupSet(setName); - assertTrue(tnames != null); - assertTrue(tnames.size() == tables.length - 2); - for (int i = 0; i < tnames.size(); i++) { - assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1))); - } - cleanBackupTable(); - } - } - - @Test - public void testBackupSetDelete() throws IOException { - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - String[] tables = new String[] { "table1", "table2", "table3", "table4" }; - String setName = "name"; - table.addToBackupSet(setName, tables); - table.deleteBackupSet(setName); - - List tnames = table.describeBackupSet(setName); - assertTrue(tnames == null); - cleanBackupTable(); - } - } - - @Test - public void testBackupSetList() throws IOException { - try (BackupSystemTable table = new BackupSystemTable(conn)) { - - String[] tables = new String[] { "table1", "table2", "table3", "table4" }; - String setName1 = "name1"; - String setName2 = "name2"; - table.addToBackupSet(setName1, tables); - table.addToBackupSet(setName2, tables); - - List list = table.listBackupSets(); - - assertTrue(list.size() == 2); - assertTrue(list.get(0).equals(setName1)); - assertTrue(list.get(1).equals(setName2)); - - cleanBackupTable(); - } - } - - private boolean compare(BackupInfo one, BackupInfo two) { - return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType()) - && one.getBackupRootDir().equals(two.getBackupRootDir()) - && one.getStartTs() == two.getStartTs() && one.getCompleteTs() == two.getCompleteTs(); - } - - private BackupInfo createBackupInfo() { - - BackupInfo ctxt = - new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL, new TableName[] { - TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") }, - "/hbase/backup"); - ctxt.setStartTs(System.currentTimeMillis()); - ctxt.setCompleteTs(System.currentTimeMillis() + 1); - return ctxt; - } - - private List createBackupInfoList(int size) { - List list = new ArrayList(); - for (int i = 0; i < size; i++) { - list.add(createBackupInfo()); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - return list; - } - - @AfterClass - public static void tearDown() throws IOException { - if (cluster != null) cluster.shutdown(); - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java deleted file mode 100644 index 919668e2b3a..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestFullBackup extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestFullBackup.class); - - @Test - public void testFullBackupMultipleCommand() throws Exception { - LOG.info("test full backup on a multiple tables with data: command-line"); - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - int before = table.getBackupHistory().size(); - String[] args = - new String[] { "create", "full", BACKUP_ROOT_DIR, "-t", - table1.getNameAsString() + "," + table2.getNameAsString() }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - List backups = table.getBackupHistory(); - int after = table.getBackupHistory().size(); - assertTrue(after == before + 1); - for (BackupInfo data : backups) { - String backupId = data.getBackupId(); - assertTrue(checkSucceeded(backupId)); - } - } - LOG.info("backup complete"); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java deleted file mode 100644 index c11597146c8..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestFullBackupSet extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestFullBackupSet.class); - - /** - * Verify that full backup is created on a single table with data correctly. - * @throws Exception - */ - @Test - public void testFullBackupSetExist() throws Exception { - - LOG.info("Test full backup, backup set exists"); - - // Create set - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - String name = "name"; - table.addToBackupSet(name, new String[] { table1.getNameAsString() }); - List names = table.describeBackupSet(name); - - assertNotNull(names); - assertTrue(names.size() == 1); - assertTrue(names.get(0).equals(table1)); - - String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - List backups = table.getBackupHistory(); - assertTrue(backups.size() == 1); - String backupId = backups.get(0).getBackupId(); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - - // Restore from set into other table - args = - new String[] { BACKUP_ROOT_DIR, backupId, "-s", name, "-m", - table1_restore.getNameAsString(), "-o" }; - // Run backup - ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret == 0); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1_restore)); - // Verify number of rows in both tables - assertEquals(TEST_UTIL.countRows(table1), TEST_UTIL.countRows(table1_restore)); - TEST_UTIL.deleteTable(table1_restore); - LOG.info("restore into other table is complete"); - hba.close(); - - } - - } - - @Test - public void testFullBackupSetDoesNotExist() throws Exception { - - LOG.info("test full backup, backup set does not exist"); - String name = "name1"; - String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret != 0); - - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java deleted file mode 100644 index 2ca13650c88..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestFullBackupSetRestoreSet extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestFullBackupSetRestoreSet.class); - - @Test - public void testFullRestoreSetToOtherTable() throws Exception { - - LOG.info("Test full restore set"); - - // Create set - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - String name = "name"; - table.addToBackupSet(name, new String[] { table1.getNameAsString() }); - List names = table.describeBackupSet(name); - - assertNotNull(names); - assertTrue(names.size() == 1); - assertTrue(names.get(0).equals(table1)); - - String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - List backups = table.getBackupHistory(); - assertTrue(backups.size() == 1); - String backupId = backups.get(0).getBackupId(); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - - // Restore from set into other table - args = - new String[] { BACKUP_ROOT_DIR, backupId, "-s", name, "-m", - table1_restore.getNameAsString(), "-o" }; - // Run backup - ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret == 0); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1_restore)); - // Verify number of rows in both tables - assertEquals(TEST_UTIL.countRows(table1), TEST_UTIL.countRows(table1_restore)); - TEST_UTIL.deleteTable(table1_restore); - LOG.info("restore into other table is complete"); - hba.close(); - } - } - - @Test - public void testFullRestoreSetToSameTable() throws Exception { - - LOG.info("Test full restore set to same table"); - - // Create set - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - String name = "name1"; - table.addToBackupSet(name, new String[] { table1.getNameAsString() }); - List names = table.describeBackupSet(name); - - assertNotNull(names); - assertTrue(names.size() == 1); - assertTrue(names.get(0).equals(table1)); - - String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - List backups = table.getBackupHistory(); - String backupId = backups.get(0).getBackupId(); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - int count = TEST_UTIL.countRows(table1); - TEST_UTIL.deleteTable(table1); - - // Restore from set into other table - args = new String[] { BACKUP_ROOT_DIR, backupId, "-s", name, "-o" }; - // Run backup - ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret == 0); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1)); - // Verify number of rows in both tables - assertEquals(count, TEST_UTIL.countRows(table1)); - LOG.info("restore into same table is complete"); - hba.close(); - - } - - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java deleted file mode 100644 index 32f40e29b9e..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.List; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.impl.TableBackupClient; -import org.apache.hadoop.hbase.backup.impl.TableBackupClient.Stage; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestFullBackupWithFailures extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestFullBackupWithFailures.class); - - @Test - public void testFullBackupWithFailures() throws Exception { - conf1.set(TableBackupClient.BACKUP_CLIENT_IMPL_CLASS, - FullTableBackupClientForTest.class.getName()); - int maxStage = Stage.values().length -1; - // Fail stages between 0 and 4 inclusive - for (int stage = 0; stage <= maxStage; stage++) { - LOG.info("Running stage " + stage); - runBackupAndFailAtStage(stage); - } - } - - public void runBackupAndFailAtStage(int stage) throws Exception { - - conf1.setInt(FullTableBackupClientForTest.BACKUP_TEST_MODE_STAGE, stage); - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - int before = table.getBackupHistory().size(); - String[] args = - new String[] { "create", "full", BACKUP_ROOT_DIR, "-t", - table1.getNameAsString() + "," + table2.getNameAsString() }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertFalse(ret == 0); - List backups = table.getBackupHistory(); - int after = table.getBackupHistory().size(); - - assertTrue(after == before +1); - for (BackupInfo data : backups) { - String backupId = data.getBackupId(); - assertFalse(checkSucceeded(backupId)); - } - Set tables = table.getIncrementalBackupTableSet(BACKUP_ROOT_DIR); - assertTrue(tables.size() == 0); - } - } - - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java deleted file mode 100644 index 3ca79f6763b..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java +++ /dev/null @@ -1,345 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestFullRestore extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestFullRestore.class); - - /** - * Verify that a single table is restored to a new table - * @throws Exception - */ - @Test - public void testFullRestoreSingle() throws Exception { - - LOG.info("test full restore on a single table empty table"); - - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - - TableName[] tableset = new TableName[] { table1 }; - TableName[] tablemap = new TableName[] { table1_restore }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - tableset, tablemap, false)); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1_restore)); - TEST_UTIL.deleteTable(table1_restore); - hba.close(); - } - - - - @Test - public void testFullRestoreSingleCommand() throws Exception { - - LOG.info("test full restore on a single table empty table: command-line"); - - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - // restore [tableMapping] - String[] args = - new String[] { BACKUP_ROOT_DIR, backupId, "-t", table1.getNameAsString(), "-m", - table1_restore.getNameAsString() }; - // Run backup - int ret = ToolRunner.run(conf1, new RestoreDriver(), args); - - assertTrue(ret == 0); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1_restore)); - TEST_UTIL.deleteTable(table1_restore); - hba.close(); - } - - @Test - public void testFullRestoreCheckCommand() throws Exception { - - LOG.info("test full restore on a single table: command-line, check only"); - - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - // restore [tableMapping] - String[] args = - new String[] { BACKUP_ROOT_DIR, backupId, "-t", table1.getNameAsString(), "-m", - table1_restore.getNameAsString(), "-c" }; - // Run backup - int ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret == 0); - //Verify that table has not been restored - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertFalse(hba.tableExists(table1_restore)); - } - - /** - * Verify that multiple tables are restored to new tables. - * @throws Exception - */ - @Test - public void testFullRestoreMultiple() throws Exception { - LOG.info("create full backup image on multiple tables"); - List tables = Lists.newArrayList(table2, table3); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - TableName[] restore_tableset = new TableName[] { table2, table3 }; - TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - restore_tableset, tablemap, false)); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table2_restore)); - assertTrue(hba.tableExists(table3_restore)); - TEST_UTIL.deleteTable(table2_restore); - TEST_UTIL.deleteTable(table3_restore); - hba.close(); - } - - /** - * Verify that multiple tables are restored to new tables. - * @throws Exception - */ - @Test - public void testFullRestoreMultipleCommand() throws Exception { - LOG.info("create full backup image on multiple tables: command-line"); - List tables = Lists.newArrayList(table2, table3); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - TableName[] restore_tableset = new TableName[] { table2, table3 }; - TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - - // restore [tableMapping] - String[] args = - new String[] { BACKUP_ROOT_DIR, backupId, "-t", StringUtils.join(restore_tableset, ","), - "-m", StringUtils.join(tablemap, ",") }; - // Run backup - int ret = ToolRunner.run(conf1, new RestoreDriver(), args); - - assertTrue(ret == 0); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table2_restore)); - assertTrue(hba.tableExists(table3_restore)); - TEST_UTIL.deleteTable(table2_restore); - TEST_UTIL.deleteTable(table3_restore); - hba.close(); - } - - /** - * Verify that a single table is restored using overwrite - * @throws Exception - */ - @Test - public void testFullRestoreSingleOverwrite() throws Exception { - - LOG.info("test full restore on a single table empty table"); - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - - TableName[] tableset = new TableName[] { table1 }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - tableset, null, true)); - } - - /** - * Verify that a single table is restored using overwrite - * @throws Exception - */ - @Test - public void testFullRestoreSingleOverwriteCommand() throws Exception { - - LOG.info("test full restore on a single table empty table: command-line"); - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - LOG.info("backup complete"); - TableName[] tableset = new TableName[] { table1 }; - // restore [tableMapping] - String[] args = - new String[] { BACKUP_ROOT_DIR, backupId, "-t", StringUtils.join(tableset, ","), "-o" }; - // Run restore - int ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret == 0); - - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1)); - hba.close(); - - } - - /** - * Verify that multiple tables are restored to new tables using overwrite. - * @throws Exception - */ - @Test - public void testFullRestoreMultipleOverwrite() throws Exception { - LOG.info("create full backup image on multiple tables"); - - List tables = Lists.newArrayList(table2, table3); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - TableName[] restore_tableset = new TableName[] { table2, table3 }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - restore_tableset, null, true)); - } - - /** - * Verify that multiple tables are restored to new tables using overwrite. - * @throws Exception - */ - @Test - public void testFullRestoreMultipleOverwriteCommand() throws Exception { - LOG.info("create full backup image on multiple tables: command-line"); - - List tables = Lists.newArrayList(table2, table3); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - TableName[] restore_tableset = new TableName[] { table2, table3 }; - // restore [tableMapping] - String[] args = - new String[] { BACKUP_ROOT_DIR, backupId, "-t", - StringUtils.join(restore_tableset, ","), "-o" }; - // Run backup - int ret = ToolRunner.run(conf1, new RestoreDriver(), args); - - assertTrue(ret == 0); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table2)); - assertTrue(hba.tableExists(table3)); - hba.close(); - } - - /** - * Verify that restore fails on a single table that does not exist. - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFullRestoreSingleDNE() throws Exception { - - LOG.info("test restore fails on a single table that does not exist"); - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - - TableName[] tableset = new TableName[] { TableName.valueOf("faketable") }; - TableName[] tablemap = new TableName[] { table1_restore }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - tableset, tablemap, false)); - } - - /** - * Verify that restore fails on a single table that does not exist. - * @throws Exception - */ - @Test - public void testFullRestoreSingleDNECommand() throws Exception { - - LOG.info("test restore fails on a single table that does not exist: command-line"); - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - - TableName[] tableset = new TableName[] { TableName.valueOf("faketable") }; - TableName[] tablemap = new TableName[] { table1_restore }; - String[] args = - new String[] { BACKUP_ROOT_DIR, backupId, StringUtils.join(tableset, ","), "-m", - StringUtils.join(tablemap, ",") }; - // Run restore - int ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret != 0); - - } - - /** - * Verify that restore fails on multiple tables that do not exist. - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFullRestoreMultipleDNE() throws Exception { - - LOG.info("test restore fails on multiple tables that do not exist"); - - List tables = Lists.newArrayList(table2, table3); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - TableName[] restore_tableset = - new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") }; - TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - restore_tableset, tablemap, false)); - } - - /** - * Verify that restore fails on multiple tables that do not exist. - * @throws Exception - */ - @Test - public void testFullRestoreMultipleDNECommand() throws Exception { - - LOG.info("test restore fails on multiple tables that do not exist: command-line"); - - List tables = Lists.newArrayList(table2, table3); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - TableName[] restore_tableset = - new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") }; - TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - String[] args = - new String[] { BACKUP_ROOT_DIR, backupId, StringUtils.join(restore_tableset, ","), "-m", - StringUtils.join(tablemap, ",") }; - // Run restore - int ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret != 0); - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java deleted file mode 100644 index 73598f30384..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -@RunWith(Parameterized.class) -public class TestIncrementalBackup extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestIncrementalBackup.class); - - @Parameterized.Parameters - public static Collection data() { - provider = "multiwal"; - List params = new ArrayList(); - params.add(new Object[] { Boolean.TRUE }); - return params; - } - - public TestIncrementalBackup(Boolean b) { - } - - // implement all test cases in 1 test since incremental backup/restore has dependencies - @Test - public void TestIncBackupRestore() throws Exception { - - int ADD_ROWS = 99; - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - - List tables = Lists.newArrayList(table1, table2); - final byte[] fam3Name = Bytes.toBytes("f3"); - table1Desc.addFamily(new HColumnDescriptor(fam3Name)); - HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc); - - Connection conn = ConnectionFactory.createConnection(conf1); - int NB_ROWS_FAM3 = 6; - insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close(); - - HBaseAdmin admin = null; - admin = (HBaseAdmin) conn.getAdmin(); - BackupAdminImpl client = new BackupAdminImpl(conn); - - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); - - assertTrue(checkSucceeded(backupIdFull)); - - // #2 - insert some data to table - HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); - LOG.debug("writing " + ADD_ROWS + " rows to " + table1); - - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3); - t1.close(); - LOG.debug("written " + ADD_ROWS + " rows to " + table1); - - HTable t2 = (HTable) conn.getTable(table2); - Put p2; - for (int i = 0; i < 5; i++) { - p2 = new Put(Bytes.toBytes("row-t2" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - - Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + 5); - t2.close(); - LOG.debug("written " + 5 + " rows to " + table2); - // split table1 - MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); - List regions = cluster.getRegions(table1); - - byte[] name = regions.get(0).getRegionInfo().getRegionName(); - long startSplitTime = EnvironmentEdgeManager.currentTime(); - try { - admin.splitRegion(name); - } catch (IOException e) { - //although split fail, this may not affect following check - //In old split without AM2, if region's best split key is not found, - //there are not exception thrown. But in current API, exception - //will be thrown. - LOG.debug("region is not splittable, because " + e); - } - - while (!admin.isTableAvailable(table1)) { - Thread.sleep(100); - } - - long endSplitTime = EnvironmentEdgeManager.currentTime(); - - // split finished - LOG.debug("split finished in =" + (endSplitTime - startSplitTime)); - - // #3 - incremental backup for multiple tables - tables = Lists.newArrayList(table1, table2); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple)); - - // add column family f2 to table1 - final byte[] fam2Name = Bytes.toBytes("f2"); - table1Desc.addFamily(new HColumnDescriptor(fam2Name)); - // drop column family f3 - table1Desc.removeFamily(fam3Name); - HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc); - - int NB_ROWS_FAM2 = 7; - HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2); - t3.close(); - - // #3 - incremental backup for multiple tables - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple2 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple2)); - - // #4 - restore full backup for all tables, without overwrite - TableName[] tablesRestoreFull = new TableName[] { table1, table2 }; - - TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore }; - - LOG.debug("Restoring full " + backupIdFull); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false, - tablesRestoreFull, tablesMapFull, false)); - - // #5.1 - check tables for full restore - HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin(); - assertTrue(hAdmin.tableExists(table1_restore)); - assertTrue(hAdmin.tableExists(table2_restore)); - - hAdmin.close(); - - // #5.2 - checking row count of tables for full restore - HTable hTable = (HTable) conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + NB_ROWS_FAM3); - hTable.close(); - - hTable = (HTable) conn.getTable(table2_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH); - hTable.close(); - - // #6 - restore incremental backup for multiple tables, with overwrite - TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 }; - TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore }; - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, - false, tablesRestoreIncMultiple, tablesMapIncMultiple, true)); - - hTable = (HTable) conn.getTable(table1_restore); - LOG.debug("After incremental restore: " + hTable.getDescriptor()); - LOG.debug("f1 has " + TEST_UTIL.countRows(hTable, famName) + " rows"); - Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH + ADD_ROWS); - LOG.debug("f2 has " + TEST_UTIL.countRows(hTable, fam2Name) + " rows"); - Assert.assertEquals(TEST_UTIL.countRows(hTable, fam2Name), NB_ROWS_FAM2); - hTable.close(); - - hTable = (HTable) conn.getTable(table2_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 5); - hTable.close(); - - admin.close(); - conn.close(); - - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java deleted file mode 100644 index 747c1ddb16e..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -/** - * 1. Create table t1, t2 - * 2. Load data to t1, t2 - * 3 Full backup t1, t2 - * 4 Delete t2 - * 5 Load data to t1 - * 6 Incremental backup t1 - */ -@Category(LargeTests.class) -public class TestIncrementalBackupDeleteTable extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestIncrementalBackupDeleteTable.class); - - // implement all test cases in 1 test since incremental backup/restore has dependencies - @Test - public void testIncBackupDeleteTable() throws Exception { - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - - List tables = Lists.newArrayList(table1, table2); - HBaseAdmin admin = null; - Connection conn = ConnectionFactory.createConnection(conf1); - admin = (HBaseAdmin) conn.getAdmin(); - BackupAdminImpl client = new BackupAdminImpl(conn); - - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); - - assertTrue(checkSucceeded(backupIdFull)); - - // #2 - insert some data to table table1 - HTable t1 = (HTable) conn.getTable(table1); - Put p1; - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH * 2); - t1.close(); - - // Delete table table2 - admin.disableTable(table2); - admin.deleteTable(table2); - - // #3 - incremental backup for table1 - tables = Lists.newArrayList(table1); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple)); - - // #4 - restore full backup for all tables, without overwrite - TableName[] tablesRestoreFull = new TableName[] { table1, table2 }; - - TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore }; - - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false, - tablesRestoreFull, tablesMapFull, false)); - - // #5.1 - check tables for full restore - HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin(); - assertTrue(hAdmin.tableExists(table1_restore)); - assertTrue(hAdmin.tableExists(table2_restore)); - - // #5.2 - checking row count of tables for full restore - HTable hTable = (HTable) conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH); - hTable.close(); - - hTable = (HTable) conn.getTable(table2_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH); - hTable.close(); - - // #6 - restore incremental backup for table1 - TableName[] tablesRestoreIncMultiple = new TableName[] { table1 }; - TableName[] tablesMapIncMultiple = new TableName[] { table1_restore }; - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple, - false, tablesRestoreIncMultiple, tablesMapIncMultiple, true)); - - hTable = (HTable) conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH * 2); - hTable.close(); - admin.close(); - conn.close(); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java deleted file mode 100644 index 011ddf29ad3..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java +++ /dev/null @@ -1,336 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupMergeJob; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestIncrementalBackupMergeWithFailures extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestIncrementalBackupMergeWithFailures.class); - - static enum FailurePhase { - PHASE1, PHASE2, PHASE3, PHASE4 - } - public final static String FAILURE_PHASE_KEY = "failurePhase"; - - static class BackupMergeJobWithFailures extends MapReduceBackupMergeJob { - - FailurePhase failurePhase; - - @Override - public void setConf(Configuration conf) { - super.setConf(conf); - String val = conf.get(FAILURE_PHASE_KEY); - if (val != null) { - failurePhase = FailurePhase.valueOf(val); - } else { - Assert.fail("Failure phase is not set"); - } - } - - - /** - * This is the exact copy of parent's run() with injections - * of different types of failures - */ - @Override - public void run(String[] backupIds) throws IOException { - String bulkOutputConfKey; - - // TODO : run player on remote cluster - player = new MapReduceHFileSplitterJob(); - bulkOutputConfKey = MapReduceHFileSplitterJob.BULK_OUTPUT_CONF_KEY; - // Player reads all files in arbitrary directory structure and creates - // a Map task for each file - String bids = StringUtils.join(backupIds, ","); - - if (LOG.isDebugEnabled()) { - LOG.debug("Merge backup images " + bids); - } - - List> processedTableList = new ArrayList>(); - boolean finishedTables = false; - Connection conn = ConnectionFactory.createConnection(getConf()); - BackupSystemTable table = new BackupSystemTable(conn); - FileSystem fs = FileSystem.get(getConf()); - - try { - - // Start backup exclusive operation - table.startBackupExclusiveOperation(); - // Start merge operation - table.startMergeOperation(backupIds); - - // Select most recent backup id - String mergedBackupId = findMostRecentBackupId(backupIds); - - TableName[] tableNames = getTableNamesInBackupImages(backupIds); - String backupRoot = null; - - BackupInfo bInfo = table.readBackupInfo(backupIds[0]); - backupRoot = bInfo.getBackupRootDir(); - // PHASE 1 - checkFailure(FailurePhase.PHASE1); - - for (int i = 0; i < tableNames.length; i++) { - - LOG.info("Merge backup images for " + tableNames[i]); - - // Find input directories for table - - Path[] dirPaths = findInputDirectories(fs, backupRoot, tableNames[i], backupIds); - String dirs = StringUtils.join(dirPaths, ","); - Path bulkOutputPath = - BackupUtils.getBulkOutputDir(BackupUtils.getFileNameCompatibleString(tableNames[i]), - getConf(), false); - // Delete content if exists - if (fs.exists(bulkOutputPath)) { - if (!fs.delete(bulkOutputPath, true)) { - LOG.warn("Can not delete: " + bulkOutputPath); - } - } - Configuration conf = getConf(); - conf.set(bulkOutputConfKey, bulkOutputPath.toString()); - String[] playerArgs = { dirs, tableNames[i].getNameAsString() }; - - int result = 0; - // PHASE 2 - checkFailure(FailurePhase.PHASE2); - player.setConf(getConf()); - result = player.run(playerArgs); - if (succeeded(result)) { - // Add to processed table list - processedTableList.add(new Pair(tableNames[i], bulkOutputPath)); - } else { - throw new IOException("Can not merge backup images for " + dirs - + " (check Hadoop/MR and HBase logs). Player return code =" + result); - } - LOG.debug("Merge Job finished:" + result); - } - List tableList = toTableNameList(processedTableList); - // PHASE 3 - checkFailure(FailurePhase.PHASE3); - table.updateProcessedTablesForMerge(tableList); - finishedTables = true; - - // Move data - for (Pair tn : processedTableList) { - moveData(fs, backupRoot, tn.getSecond(), tn.getFirst(), mergedBackupId); - } - // PHASE 4 - checkFailure(FailurePhase.PHASE4); - // Delete old data and update manifest - List backupsToDelete = getBackupIdsToDelete(backupIds, mergedBackupId); - deleteBackupImages(backupsToDelete, conn, fs, backupRoot); - updateBackupManifest(backupRoot, mergedBackupId, backupsToDelete); - // Finish merge session - table.finishMergeOperation(); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - LOG.error(e); - if (!finishedTables) { - // cleanup bulk directories and finish merge - // merge MUST be repeated (no need for repair) - cleanupBulkLoadDirs(fs, toPathList(processedTableList)); - table.finishMergeOperation(); - table.finishBackupExclusiveOperation(); - throw new IOException("Backup merge operation failed, you should try it again", e); - } else { - // backup repair must be run - throw new IOException( - "Backup merge operation failed, run backup repair tool to restore system's integrity", - e); - } - } finally { - table.close(); - conn.close(); - } - - } - - private void checkFailure(FailurePhase phase) throws IOException { - if ( failurePhase != null && failurePhase == phase) { - throw new IOException (phase.toString()); - } - } - - } - - - @Test - public void TestIncBackupMergeRestore() throws Exception { - - int ADD_ROWS = 99; - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - - List tables = Lists.newArrayList(table1, table2); - // Set custom Merge Job implementation - conf1.setClass(BackupRestoreFactory.HBASE_BACKUP_MERGE_IMPL_CLASS, - BackupMergeJobWithFailures.class, BackupMergeJob.class); - - Connection conn = ConnectionFactory.createConnection(conf1); - - HBaseAdmin admin = null; - admin = (HBaseAdmin) conn.getAdmin(); - BackupAdminImpl client = new BackupAdminImpl(conn); - - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); - - assertTrue(checkSucceeded(backupIdFull)); - - // #2 - insert some data to table1 - HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); - LOG.debug("writing " + ADD_ROWS + " rows to " + table1); - - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS); - t1.close(); - LOG.debug("written " + ADD_ROWS + " rows to " + table1); - - HTable t2 = insertIntoTable(conn, table2, famName, 1, ADD_ROWS); - - Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + ADD_ROWS); - t2.close(); - LOG.debug("written " + ADD_ROWS + " rows to " + table2); - - // #3 - incremental backup for multiple tables - tables = Lists.newArrayList(table1, table2); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); - - assertTrue(checkSucceeded(backupIdIncMultiple)); - - t1 = insertIntoTable(conn, table1, famName, 2, ADD_ROWS); - t1.close(); - - t2 = insertIntoTable(conn, table2, famName, 2, ADD_ROWS); - t2.close(); - - // #3 - incremental backup for multiple tables - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple2 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple2)); - - // #4 Merge backup images with failures - - for ( FailurePhase phase : FailurePhase.values()) { - Configuration conf = conn.getConfiguration(); - - conf.set(FAILURE_PHASE_KEY, phase.toString()); - - try (BackupAdmin bAdmin = new BackupAdminImpl(conn);) - { - String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 }; - bAdmin.mergeBackups(backups); - Assert.fail("Expected IOException"); - } catch (IOException e) { - BackupSystemTable table = new BackupSystemTable(conn); - if(phase.ordinal() < FailurePhase.PHASE4.ordinal()) { - // No need to repair: - // Both Merge and backup exclusive operations are finished - assertFalse(table.isMergeInProgress()); - try { - table.finishBackupExclusiveOperation(); - Assert.fail("IOException is expected"); - } catch(IOException ee) { - // Expected - } - } else { - // Repair is required - assertTrue(table.isMergeInProgress()); - try { - table.startBackupExclusiveOperation(); - Assert.fail("IOException is expected"); - } catch(IOException ee) { - // Expected - clean up before proceeding - table.finishMergeOperation(); - table.finishBackupExclusiveOperation(); - } - } - table.close(); - LOG.debug("Expected :"+ e.getMessage()); - } - } - - // Now merge w/o failures - Configuration conf = conn.getConfiguration(); - conf.unset(FAILURE_PHASE_KEY); - conf.unset(BackupRestoreFactory.HBASE_BACKUP_MERGE_IMPL_CLASS); - - try (BackupAdmin bAdmin = new BackupAdminImpl(conn);) { - String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 }; - bAdmin.mergeBackups(backups); - } - - // #6 - restore incremental backup for multiple tables, with overwrite - TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 }; - TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore }; - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, false, - tablesRestoreIncMultiple, tablesMapIncMultiple, true)); - - Table hTable = conn.getTable(table1_restore); - LOG.debug("After incremental restore: " + hTable.getTableDescriptor()); - LOG.debug("f1 has " + TEST_UTIL.countRows(hTable, famName) + " rows"); - Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH + 2 * ADD_ROWS); - - hTable.close(); - - hTable = conn.getTable(table2_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 2 * ADD_ROWS); - hTable.close(); - - admin.close(); - conn.close(); - - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java deleted file mode 100644 index f63bf298b60..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.tool.TestLoadIncrementalHFiles; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -/** - * 1. Create table t1 - * 2. Load data to t1 - * 3 Full backup t1 - * 4 Load data to t1 - * 5 bulk load into t1 - * 6 Incremental backup t1 - */ -@Category(LargeTests.class) -@RunWith(Parameterized.class) -public class TestIncrementalBackupWithBulkLoad extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestIncrementalBackupDeleteTable.class); - - @Parameterized.Parameters - public static Collection data() { - secure = true; - List params = new ArrayList(); - params.add(new Object[] {Boolean.TRUE}); - return params; - } - - public TestIncrementalBackupWithBulkLoad(Boolean b) { - } - // implement all test cases in 1 test since incremental backup/restore has dependencies - @Test - public void TestIncBackupDeleteTable() throws Exception { - String testName = "TestIncBackupDeleteTable"; - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - - List tables = Lists.newArrayList(table1); - HBaseAdmin admin = null; - Connection conn = ConnectionFactory.createConnection(conf1); - admin = (HBaseAdmin) conn.getAdmin(); - BackupAdminImpl client = new BackupAdminImpl(conn); - - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); - - assertTrue(checkSucceeded(backupIdFull)); - - // #2 - insert some data to table table1 - HTable t1 = (HTable) conn.getTable(table1); - Put p1; - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH * 2); - t1.close(); - - int NB_ROWS2 = 20; - LOG.debug("bulk loading into " + testName); - int actual = TestLoadIncrementalHFiles.loadHFiles(testName, table1Desc, TEST_UTIL, famName, - qualName, false, null, new byte[][][] { - new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") }, - new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") }, - }, true, false, true, NB_ROWS_IN_BATCH*2, NB_ROWS2); - - // #3 - incremental backup for table1 - tables = Lists.newArrayList(table1); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple)); - - // #5.1 - check tables for full restore */ - HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin(); - - // #6 - restore incremental backup for table1 - TableName[] tablesRestoreIncMultiple = new TableName[] { table1 }; - TableName[] tablesMapIncMultiple = new TableName[] { table1_restore }; - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple, - false, tablesRestoreIncMultiple, tablesMapIncMultiple, true)); - - HTable hTable = (HTable) conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH * 2+actual); - request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - - backupIdFull = client.backupTables(request); - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - Pair>>>>, List> pair - = table.readBulkloadRows(tables); - assertTrue("map still has " + pair.getSecond().size() + " entries", - pair.getSecond().isEmpty()); - } - assertTrue(checkSucceeded(backupIdFull)); - - hTable.close(); - admin.close(); - conn.close(); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java deleted file mode 100644 index 84a596eb5c7..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.backup.impl.TableBackupClient; -import org.apache.hadoop.hbase.backup.impl.TableBackupClient.Stage; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -@RunWith(Parameterized.class) -public class TestIncrementalBackupWithFailures extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestIncrementalBackupWithFailures.class); - - @Parameterized.Parameters - public static Collection data() { - provider = "multiwal"; - List params = new ArrayList(); - params.add(new Object[] { Boolean.TRUE }); - return params; - } - - public TestIncrementalBackupWithFailures(Boolean b) { - } - - // implement all test cases in 1 test since incremental backup/restore has dependencies - @Test - public void testIncBackupRestore() throws Exception { - - int ADD_ROWS = 99; - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - - List tables = Lists.newArrayList(table1, table2); - final byte[] fam3Name = Bytes.toBytes("f3"); - table1Desc.addFamily(new HColumnDescriptor(fam3Name)); - HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc); - - Connection conn = ConnectionFactory.createConnection(conf1); - int NB_ROWS_FAM3 = 6; - insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close(); - - HBaseAdmin admin = null; - admin = (HBaseAdmin) conn.getAdmin(); - BackupAdminImpl client = new BackupAdminImpl(conn); - - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); - - assertTrue(checkSucceeded(backupIdFull)); - - // #2 - insert some data to table - HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); - LOG.debug("writing " + ADD_ROWS + " rows to " + table1); - - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3); - t1.close(); - LOG.debug("written " + ADD_ROWS + " rows to " + table1); - - HTable t2 = (HTable) conn.getTable(table2); - Put p2; - for (int i = 0; i < 5; i++) { - p2 = new Put(Bytes.toBytes("row-t2" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - - Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + 5); - t2.close(); - LOG.debug("written " + 5 + " rows to " + table2); - - // #3 - incremental backup for multiple tables - incrementalBackupWithFailures(); - - admin.close(); - conn.close(); - - } - - - private void incrementalBackupWithFailures() throws Exception { - conf1.set(TableBackupClient.BACKUP_CLIENT_IMPL_CLASS, - IncrementalTableBackupClientForTest.class.getName()); - int maxStage = Stage.values().length -1; - // Fail stages between 0 and 4 inclusive - for (int stage = 0; stage <= maxStage; stage++) { - LOG.info("Running stage " + stage); - runBackupAndFailAtStage(stage); - } - } - - private void runBackupAndFailAtStage(int stage) throws Exception { - - conf1.setInt(FullTableBackupClientForTest.BACKUP_TEST_MODE_STAGE, stage); - try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { - int before = table.getBackupHistory().size(); - String[] args = - new String[] { "create", "incremental", BACKUP_ROOT_DIR, "-t", - table1.getNameAsString() + "," + table2.getNameAsString() }; - // Run backup - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertFalse(ret == 0); - List backups = table.getBackupHistory(); - int after = table.getBackupHistory().size(); - - assertTrue(after == before +1); - for (BackupInfo data : backups) { - if(data.getType() == BackupType.FULL) { - assertTrue(data.getState() == BackupState.COMPLETE); - } else { - assertTrue(data.getState() == BackupState.FAILED); - } - } - } - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java deleted file mode 100644 index d8594c58fc3..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.concurrent.CountDownLatch; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestRemoteBackup extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestRemoteBackup.class); - - @Override - public void setUp () throws Exception { - useSecondCluster = true; - super.setUp(); - } - - /** - * Verify that a remote full backup is created on a single table with data correctly. - * @throws Exception - */ - @Test - public void testFullBackupRemote() throws Exception { - LOG.info("test remote full backup on a single table"); - final CountDownLatch latch = new CountDownLatch(1); - final int NB_ROWS_IN_FAM3 = 6; - final byte[] fam3Name = Bytes.toBytes("f3"); - final byte[] fam2Name = Bytes.toBytes("f2"); - final Connection conn = ConnectionFactory.createConnection(conf1); - Thread t = new Thread() { - @Override - public void run() { - try { - latch.await(); - } catch (InterruptedException ie) { - } - try { - HTable t1 = (HTable) conn.getTable(table1); - Put p1; - for (int i = 0; i < NB_ROWS_IN_FAM3; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(fam3Name, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - LOG.debug("Wrote " + NB_ROWS_IN_FAM3 + " rows into family3"); - t1.close(); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - }; - t.start(); - - table1Desc.addFamily(new HColumnDescriptor(fam3Name)); - // family 2 is MOB enabled - HColumnDescriptor hcd = new HColumnDescriptor(fam2Name); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0L); - table1Desc.addFamily(hcd); - HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc); - - SnapshotTestingUtils.loadData(TEST_UTIL, table1, 50, fam2Name); - HTable t1 = (HTable) conn.getTable(table1); - int rows0 = MobSnapshotTestingUtils.countMobRows(t1, fam2Name); - - latch.countDown(); - String backupId = - backupTables(BackupType.FULL, Lists.newArrayList(table1), BACKUP_REMOTE_ROOT_DIR); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete " + backupId); - Assert.assertEquals(TEST_UTIL.countRows(t1, famName), NB_ROWS_IN_BATCH); - - t.join(); - Assert.assertEquals(TEST_UTIL.countRows(t1, fam3Name), NB_ROWS_IN_FAM3); - t1.close(); - - TableName[] tablesRestoreFull = new TableName[] { table1 }; - - TableName[] tablesMapFull = new TableName[] { table1_restore }; - - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_REMOTE_ROOT_DIR, backupId, false, - tablesRestoreFull, tablesMapFull, false)); - - // check tables for full restore - HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin(); - assertTrue(hAdmin.tableExists(table1_restore)); - - // #5.2 - checking row count of tables for full restore - HTable hTable = (HTable) conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH); - int cnt3 = TEST_UTIL.countRows(hTable, fam3Name); - Assert.assertTrue(cnt3 >= 0 && cnt3 <= NB_ROWS_IN_FAM3); - - int rows1 = MobSnapshotTestingUtils.countMobRows(t1, fam2Name); - Assert.assertEquals(rows0, rows1); - hTable.close(); - - hAdmin.close(); - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java deleted file mode 100644 index f7dcb9f301f..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestRemoteRestore extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestRemoteRestore.class); - - @Override - public void setUp () throws Exception { - useSecondCluster = true; - super.setUp(); - } - - - /** - * Verify that a remote restore on a single table is successful. - * @throws Exception - */ - @Test - public void testFullRestoreRemote() throws Exception { - - LOG.info("test remote full backup on a single table"); - String backupId = - backupTables(BackupType.FULL, toList(table1.getNameAsString()), BACKUP_REMOTE_ROOT_DIR); - LOG.info("backup complete"); - TableName[] tableset = new TableName[] { table1 }; - TableName[] tablemap = new TableName[] { table1_restore }; - getBackupAdmin().restore( - BackupUtils.createRestoreRequest(BACKUP_REMOTE_ROOT_DIR, backupId, false, tableset, - tablemap, false)); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1_restore)); - TEST_UTIL.deleteTable(table1_restore); - hba.close(); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java deleted file mode 100644 index 556521f7cb6..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestRepairAfterFailedDelete extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestRepairAfterFailedDelete.class); - - @Test - public void testRepairBackupDelete() throws Exception { - LOG.info("test repair backup delete on a single table with data"); - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId)); - LOG.info("backup complete"); - String[] backupIds = new String[] { backupId }; - BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection()); - BackupInfo info = table.readBackupInfo(backupId); - Path path = new Path(info.getBackupRootDir(), backupId); - FileSystem fs = FileSystem.get(path.toUri(), conf1); - assertTrue(fs.exists(path)); - - // Snapshot backup system table before delete - String snapshotName = "snapshot-backup"; - Connection conn = TEST_UTIL.getConnection(); - Admin admin = conn.getAdmin(); - admin.snapshot(snapshotName, BackupSystemTable.getTableName(conf1)); - - int deleted = getBackupAdmin().deleteBackups(backupIds); - - assertTrue(!fs.exists(path)); - assertTrue(fs.exists(new Path(info.getBackupRootDir()))); - assertTrue(1 == deleted); - - // Emulate delete failure - // Restore backup system table - admin.disableTable(BackupSystemTable.getTableName(conf1)); - admin.restoreSnapshot(snapshotName); - admin.enableTable(BackupSystemTable.getTableName(conf1)); - // Start backup session - table.startBackupExclusiveOperation(); - // Start delete operation - table.startDeleteOperation(backupIds); - - // Now run repair command to repair "failed" delete operation - String[] args = new String[] {"repair"}; - // Run restore - int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - // Verify that history length == 0 - assertTrue (table.getBackupHistory().size() == 0); - table.close(); - admin.close(); - } - - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java deleted file mode 100644 index da23c352b69..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestRestoreBoundaryTests extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestRestoreBoundaryTests.class); - - /** - * Verify that a single empty table is restored to a new table - * @throws Exception - */ - @Test - public void testFullRestoreSingleEmpty() throws Exception { - LOG.info("test full restore on a single table empty table"); - String backupId = fullTableBackup(toList(table1.getNameAsString())); - LOG.info("backup complete"); - TableName[] tableset = new TableName[] { table1 }; - TableName[] tablemap = new TableName[] { table1_restore }; - getBackupAdmin().restore( - BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tableset, tablemap, - false)); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table1_restore)); - TEST_UTIL.deleteTable(table1_restore); - } - - /** - * Verify that multiple tables are restored to new tables. - * @throws Exception - */ - @Test - public void testFullRestoreMultipleEmpty() throws Exception { - LOG.info("create full backup image on multiple tables"); - - List tables = toList(table2.getNameAsString(), table3.getNameAsString()); - String backupId = fullTableBackup(tables); - TableName[] restore_tableset = new TableName[] { table2, table3 }; - TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - getBackupAdmin().restore( - BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, restore_tableset, - tablemap, false)); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(table2_restore)); - assertTrue(hba.tableExists(table3_restore)); - TEST_UTIL.deleteTable(table2_restore); - TEST_UTIL.deleteTable(table3_restore); - hba.close(); - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java deleted file mode 100644 index 2a0c4b46400..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestSystemTableSnapshot extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestSystemTableSnapshot.class); - - /** - * Verify backup system table snapshot - * @throws Exception - */ - // @Test - public void _testBackupRestoreSystemTable() throws Exception { - - LOG.info("test snapshot system table"); - - TableName backupSystem = BackupSystemTable.getTableName(conf1); - - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - String snapshotName = "sysTable"; - hba.snapshot(snapshotName, backupSystem); - - hba.disableTable(backupSystem); - hba.restoreSnapshot(snapshotName); - hba.enableTable(backupSystem); - hba.close(); - } - -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java deleted file mode 100644 index 5f72f458b96..00000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.master; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.TestBackupBase; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterables; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category(LargeTests.class) -public class TestBackupLogCleaner extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class); - - // implements all test cases in 1 test since incremental full backup/ - // incremental backup has dependencies - @Test - public void testBackupLogCleaner() throws Exception { - - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - - List tableSetFullList = Lists.newArrayList(table1, table2, table3, table4); - - try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { - // Verify that we have no backup sessions yet - assertFalse(systemTable.hasBackupSessions()); - - List walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration()); - List swalFiles = convert(walFiles); - BackupLogCleaner cleaner = new BackupLogCleaner(); - cleaner.setConf(TEST_UTIL.getConfiguration()); - cleaner.init(null); - cleaner.setConf(TEST_UTIL.getConfiguration()); - - Iterable deletable = cleaner.getDeletableFiles(walFiles); - int size = Iterables.size(deletable); - - // We can delete all files because we do not have yet recorded backup sessions - assertTrue(size == walFiles.size()); - - systemTable.addWALFiles(swalFiles, "backup", "root"); - String backupIdFull = fullTableBackup(tableSetFullList); - assertTrue(checkSucceeded(backupIdFull)); - // Check one more time - deletable = cleaner.getDeletableFiles(walFiles); - // We can delete wal files because they were saved into backup system table table - size = Iterables.size(deletable); - assertTrue(size == walFiles.size()); - - List newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration()); - LOG.debug("WAL list after full backup"); - convert(newWalFiles); - - // New list of wal files is greater than the previous one, - // because new wal per RS have been opened after full backup - assertTrue(walFiles.size() < newWalFiles.size()); - Connection conn = ConnectionFactory.createConnection(conf1); - // #2 - insert some data to table - HTable t1 = (HTable) conn.getTable(table1); - Put p1; - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - - t1.close(); - - HTable t2 = (HTable) conn.getTable(table2); - Put p2; - for (int i = 0; i < 5; i++) { - p2 = new Put(Bytes.toBytes("row-t2" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - - t2.close(); - - // #3 - incremental backup for multiple tables - - List tableSetIncList = Lists.newArrayList(table1, table2, table3); - String backupIdIncMultiple = backupTables(BackupType.INCREMENTAL, tableSetIncList, - BACKUP_ROOT_DIR); - assertTrue(checkSucceeded(backupIdIncMultiple)); - deletable = cleaner.getDeletableFiles(newWalFiles); - - assertTrue(Iterables.size(deletable) == newWalFiles.size()); - - conn.close(); - } - } - - private List convert(List walFiles) { - List result = new ArrayList(); - for (FileStatus fs : walFiles) { - LOG.debug("+++WAL: " + fs.getPath().toString()); - result.add(fs.getPath().toString()); - } - return result; - } - - private List getListOfWALFiles(Configuration c) throws IOException { - Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME); - FileSystem fs = FileSystem.get(c); - RemoteIterator it = fs.listFiles(logRoot, true); - List logFiles = new ArrayList(); - while (it.hasNext()) { - LocatedFileStatus lfs = it.next(); - if (lfs.isFile() && !AbstractFSWALProvider.isMetaFile(lfs.getPath())) { - logFiles.add(lfs); - LOG.info(lfs); - } - } - return logFiles; - } - -} diff --git a/hbase-backup/src/test/resources/log4j.properties b/hbase-backup/src/test/resources/log4j.properties deleted file mode 100644 index c322699ced2..00000000000 --- a/hbase-backup/src/test/resources/log4j.properties +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hbase.root.logger=INFO,console -hbase.log.dir=. -hbase.log.file=hbase.log - -# Define the root logger to the system property "hbase.root.logger". -log4j.rootLogger=${hbase.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout -# Debugging Pattern format -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n - -# Custom Logging levels - -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG - -log4j.logger.org.apache.hadoop=WARN -log4j.logger.org.apache.zookeeper=ERROR -log4j.logger.org.apache.hadoop.hbase=DEBUG - -#These settings are workarounds against spurious logs from the minicluster. -#See HBASE-4709 -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN -log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN -# Enable this to get detailed connection error/retry logging. -# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 2ab02cc69ca..058efff8c00 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -222,11 +222,6 @@ org.apache.hbase hbase-testing-util - - org.apache.hbase - hbase-backup - ${project.version} - org.apache.hbase.thirdparty hbase-shaded-miscellaneous diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java deleted file mode 100644 index f041f72f295..00000000000 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java +++ /dev/null @@ -1,313 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.List; -import java.util.Set; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.backup.BackupAdmin; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.RestoreRequest; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.testclassification.IntegrationTests; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.hbase.shaded.com.google.common.base.MoreObjects; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.shaded.com.google.common.base.Objects; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -/** - * An integration test to detect regressions in HBASE-7912. Create - * a table with many regions, load data, perform series backup/load operations, - * then restore and verify data - * @see HBASE-7912 - * @see HBASE-14123 - */ -@Category(IntegrationTests.class) -public class IntegrationTestBackupRestore extends IntegrationTestBase { - private static final String CLASS_NAME = IntegrationTestBackupRestore.class.getSimpleName(); - protected static final Log LOG = LogFactory.getLog(IntegrationTestBackupRestore.class); - protected static final TableName TABLE_NAME1 = TableName.valueOf(CLASS_NAME + ".table1"); - protected static final TableName TABLE_NAME2 = TableName.valueOf(CLASS_NAME + ".table2"); - protected static final String COLUMN_NAME = "f"; - protected static final String REGION_COUNT_KEY = "regions_per_rs"; - protected static final String REGIONSERVER_COUNT_KEY = "region_servers"; - protected static final int DEFAULT_REGION_COUNT = 10; - protected static final int DEFAULT_REGIONSERVER_COUNT = 2; - protected static int regionsCountPerServer; - protected static int regionServerCount; - protected static final String NB_ROWS_IN_BATCH_KEY = "rows_in_batch"; - protected static final int DEFAULT_NB_ROWS_IN_BATCH = 20000; - private static int rowsInBatch; - private static String BACKUP_ROOT_DIR = "backupIT"; - - @Override - @Before - public void setUp() throws Exception { - util = new IntegrationTestingUtility(); - Configuration conf = util.getConfiguration(); - regionsCountPerServer = conf.getInt(REGION_COUNT_KEY, DEFAULT_REGION_COUNT); - regionServerCount = - conf.getInt(REGIONSERVER_COUNT_KEY, DEFAULT_REGIONSERVER_COUNT); - rowsInBatch = conf.getInt(NB_ROWS_IN_BATCH_KEY, DEFAULT_NB_ROWS_IN_BATCH); - enableBackup(conf); - LOG.info(String.format("Initializing cluster with %d region servers.", regionServerCount)); - util.initializeCluster(regionServerCount); - LOG.info("Cluster initialized"); - util.deleteTableIfAny(TABLE_NAME1); - util.deleteTableIfAny(TABLE_NAME2); - LOG.info("Cluster ready"); - } - - @After - public void tearDown() throws IOException { - LOG.info("Cleaning up after test."); - if(util.isDistributedCluster()) { - util.deleteTableIfAny(TABLE_NAME1); - LOG.info("Cleaning up after test. TABLE1 done"); - util.deleteTableIfAny(TABLE_NAME2); - LOG.info("Cleaning up after test. TABLE2 done"); - cleanUpBackupDir(); - } - LOG.info("Restoring cluster."); - util.restoreCluster(); - LOG.info("Cluster restored."); - } - - private void enableBackup(Configuration conf) { - // Enable backup - conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); - BackupManager.decorateMasterConfiguration(conf); - BackupManager.decorateRegionServerConfiguration(conf); - } - - private void cleanUpBackupDir() throws IOException { - FileSystem fs = FileSystem.get(util.getConfiguration()); - fs.delete(new Path(BACKUP_ROOT_DIR), true); - } - - @Test - public void testBackupRestore() throws Exception { - BACKUP_ROOT_DIR = util.getDataTestDirOnTestFS() + Path.SEPARATOR + BACKUP_ROOT_DIR; - createTable(TABLE_NAME1); - createTable(TABLE_NAME2); - runTest(); - } - - - private void createTable(TableName tableName) throws Exception { - long startTime, endTime; - HTableDescriptor desc = new HTableDescriptor(tableName); - HColumnDescriptor[] columns = - new HColumnDescriptor[]{new HColumnDescriptor(COLUMN_NAME)}; - LOG.info(String.format("Creating table %s with %d splits.", tableName, - regionsCountPerServer)); - startTime = System.currentTimeMillis(); - HBaseTestingUtility.createPreSplitLoadTestTable(util.getConfiguration(), desc, columns, - regionsCountPerServer); - util.waitTableAvailable(tableName); - endTime = System.currentTimeMillis(); - LOG.info(String.format("Pre-split table created successfully in %dms.", - (endTime - startTime))); - } - - private void loadData(TableName table, int numRows) throws IOException { - Connection conn = util.getConnection(); - // #0- insert some data to a table - Table t1 = conn.getTable(table); - util.loadRandomRows(t1, new byte[]{'f'}, 100, numRows); - // flush table - conn.getAdmin().flush(TableName.valueOf(table.getName())); - } - - private void runTest() throws IOException { - - try (Connection conn = util.getConnection(); - Admin admin = conn.getAdmin(); - BackupAdmin client = new BackupAdminImpl(conn);) { - - // #0- insert some data to table TABLE_NAME1, TABLE_NAME2 - loadData(TABLE_NAME1, rowsInBatch); - loadData(TABLE_NAME2, rowsInBatch); - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - List tables = Lists.newArrayList(TABLE_NAME1, TABLE_NAME2); - - BackupRequest.Builder builder = new BackupRequest.Builder(); - BackupRequest request = - builder.withBackupType(BackupType.FULL).withTableList(tables) - .withTargetRootDir(BACKUP_ROOT_DIR).build(); - String backupIdFull = client.backupTables(request); - assertTrue(checkSucceeded(backupIdFull)); - // #2 - insert some data to table - loadData(TABLE_NAME1, rowsInBatch); - loadData(TABLE_NAME2, rowsInBatch); - HTable t1 = (HTable) conn.getTable(TABLE_NAME1); - Assert.assertEquals(util.countRows(t1), rowsInBatch * 2); - t1.close(); - HTable t2 = (HTable) conn.getTable(TABLE_NAME2); - Assert.assertEquals(util.countRows(t2), rowsInBatch * 2); - t2.close(); - // #3 - incremental backup for tables - tables = Lists.newArrayList(TABLE_NAME1, TABLE_NAME2); - builder = new BackupRequest.Builder(); - request = - builder.withBackupType(BackupType.INCREMENTAL).withTableList(tables) - .withTargetRootDir(BACKUP_ROOT_DIR).build(); - String backupIdIncMultiple = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple)); - // #4 - restore full backup for all tables, without overwrite - TableName[] tablesRestoreFull = new TableName[] { TABLE_NAME1, TABLE_NAME2 }; - client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false, tablesRestoreFull, - null, true)); - // #5.1 - check tables for full restore - assertTrue(admin.tableExists(TABLE_NAME1)); - assertTrue(admin.tableExists(TABLE_NAME2)); - // #5.2 - checking row count of tables for full restore - HTable hTable = (HTable) conn.getTable(TABLE_NAME1); - Assert.assertEquals(util.countRows(hTable), rowsInBatch); - hTable.close(); - hTable = (HTable) conn.getTable(TABLE_NAME2); - Assert.assertEquals(util.countRows(hTable), rowsInBatch); - hTable.close(); - // #6 - restore incremental backup for multiple tables, with overwrite - TableName[] tablesRestoreIncMultiple = new TableName[] { TABLE_NAME1, TABLE_NAME2 }; - client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple, false, - tablesRestoreIncMultiple, null, true)); - hTable = (HTable) conn.getTable(TABLE_NAME1); - Assert.assertEquals(util.countRows(hTable), rowsInBatch * 2); - hTable.close(); - hTable = (HTable) conn.getTable(TABLE_NAME2); - Assert.assertEquals(util.countRows(hTable), rowsInBatch * 2); - hTable.close(); - } - } - - protected boolean checkSucceeded(String backupId) throws IOException { - BackupInfo status = getBackupInfo(backupId); - if (status == null) return false; - return status.getState() == BackupState.COMPLETE; - } - - private BackupInfo getBackupInfo(String backupId) throws IOException { - try (BackupSystemTable table = new BackupSystemTable(util.getConnection())) { - return table.readBackupInfo(backupId); - } - } - - /** - * Get restore request. - */ - public RestoreRequest createRestoreRequest(String backupRootDir, String backupId, boolean check, - TableName[] fromTables, TableName[] toTables, boolean isOverwrite) { - RestoreRequest.Builder builder = new RestoreRequest.Builder(); - return builder.withBackupRootDir(backupRootDir) - .withBackupId(backupId) - .withCheck(check) - .withFromTables(fromTables) - .withToTables(toTables) - .withOvewrite(isOverwrite).build(); - } - - @Override - public void setUpCluster() throws Exception { - util = getTestingUtil(getConf()); - enableBackup(getConf()); - LOG.debug("Initializing/checking cluster has " + regionServerCount + " servers"); - util.initializeCluster(regionServerCount); - LOG.debug("Done initializing/checking cluster"); - } - - @Override - public int runTestFromCommandLine() throws Exception { - testBackupRestore(); - return 0; - } - - @Override - public TableName getTablename() { - // That is only valid when Monkey is CALM (no monkey) - return null; - } - - @Override - protected Set getColumnFamilies() { - // That is only valid when Monkey is CALM (no monkey) - return null; - } - - @Override - protected void addOptions() { - addOptWithArg(REGIONSERVER_COUNT_KEY, "Total number of region servers. Default: '" - + DEFAULT_REGIONSERVER_COUNT + "'"); - addOptWithArg(REGION_COUNT_KEY, "Total number of regions. Default: " + DEFAULT_REGION_COUNT); - addOptWithArg(NB_ROWS_IN_BATCH_KEY, "Total number of data rows to be loaded (per table/batch." - + " Total number of batches=2). Default: " + DEFAULT_NB_ROWS_IN_BATCH); - - } - - @Override - protected void processOptions(CommandLine cmd) { - super.processOptions(cmd); - regionsCountPerServer = - Integer.parseInt(cmd.getOptionValue(REGION_COUNT_KEY, - Integer.toString(DEFAULT_REGION_COUNT))); - regionServerCount = - Integer.parseInt(cmd.getOptionValue(REGIONSERVER_COUNT_KEY, - Integer.toString(DEFAULT_REGIONSERVER_COUNT))); - rowsInBatch = - Integer.parseInt(cmd.getOptionValue(NB_ROWS_IN_BATCH_KEY, - Integer.toString(DEFAULT_NB_ROWS_IN_BATCH))); - LOG.info(MoreObjects.toStringHelper("Parsed Options"). - add(REGION_COUNT_KEY, regionsCountPerServer) - .add(REGIONSERVER_COUNT_KEY, regionServerCount).add(NB_ROWS_IN_BATCH_KEY, rowsInBatch) - .toString()); - } - - public static void main(String[] args) throws Exception { - Configuration conf = HBaseConfiguration.create(); - IntegrationTestingUtility.setUseDistributedCluster(conf); - int status = ToolRunner.run(conf, new IntegrationTestBackupRestore(), args); - System.exit(status); - } -} diff --git a/hbase-protocol-shaded/src/main/protobuf/Backup.proto b/hbase-protocol-shaded/src/main/protobuf/Backup.proto deleted file mode 100644 index 6084e17bcd2..00000000000 --- a/hbase-protocol-shaded/src/main/protobuf/Backup.proto +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless optional by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto2"; -// This file contains Backup manifest -package hbase.pb; - -option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; -option java_outer_classname = "BackupProtos"; -option java_generic_services = true; -option java_generate_equals_and_hash = true; -option optimize_for = SPEED; - -import "HBase.proto"; - -/** - * Backup type enum: FULL or INCREMENTAL - */ - -enum BackupType { - FULL = 0; - INCREMENTAL = 1; -} -/** - * ServerTimestamp keeps last WAL roll time per Region Server - */ -message ServerTimestamp { - optional ServerName server_name = 1; - optional uint64 timestamp = 2; -} - -/** - * TableServerTimestamp keeps last WAL roll time per Region Server & Table - * Each table have different last WAL roll time stamps across cluster, on every RS - */ -message TableServerTimestamp { - optional TableName table_name = 1; - repeated ServerTimestamp server_timestamp = 2; -} - -/** - * Structure keeps relevant info for backup restore session - */ -message BackupImage { - optional string backup_id = 1; - optional BackupType backup_type = 2; - optional string backup_root_dir = 3; - repeated TableName table_list = 4; - optional uint64 start_ts = 5; - optional uint64 complete_ts = 6; - repeated BackupImage ancestors = 7; - repeated TableServerTimestamp tst_map = 8; - -} - -/** - * Internal structure used during backup - */ -message BackupTableInfo { - optional TableName table_name = 1; - optional string snapshot_name = 2; -} - -/** - * Backup session information - */ -message BackupInfo { - optional string backup_id = 1; - optional BackupType backup_type = 2; - optional string backup_root_dir = 3; - optional BackupState backup_state = 4; - optional BackupPhase backup_phase = 5; - optional string failed_message = 6; - repeated BackupTableInfo backup_table_info = 7; - optional uint64 start_ts = 8; - optional uint64 complete_ts = 9; - optional uint32 progress = 10; - optional uint32 workers_number = 11; - optional uint64 bandwidth = 12; - - /** - * Backup session states - */ - enum BackupState { - RUNNING = 0; - COMPLETE = 1; - FAILED = 2; - CANCELLED = 3; - } - - /** - * Phases of a backup session in RUNNING state - * - */ - enum BackupPhase { - REQUEST = 0; - SNAPSHOT = 1; - PREPARE_INCREMENTAL = 2; - SNAPSHOTCOPY = 3; - INCREMENTAL_COPY = 4; - STORE_MANIFEST = 5; - } -} diff --git a/pom.xml b/pom.xml index df212381e7b..832e1b77642 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,6 @@ hbase-archetypes hbase-metrics-api hbase-metrics - hbase-backup hbase-zookeeper - - org.apache.hbase - hbase-backup - ${project.version} - org.apache.hbase hbase-common