HBASE-20656 Validate pre-2.0 coprocessors against HBase 2.0+
Signed-off-by: Mike Drob <mdrob@apache.org>
This commit is contained in:
Balazs Meszaros
Mike Drob
parent
eb13cdd7ed
commit
c323e7bfaa
|
|
@ -38,4 +38,5 @@
|
||||||
<suppress checks="InterfaceIsTypeCheck" files=".*/src/main/.*\.java"/>
|
<suppress checks="InterfaceIsTypeCheck" files=".*/src/main/.*\.java"/>
|
||||||
<suppress checks="EmptyBlockCheck" files="TBoundedThreadPoolServer.java"/>
|
<suppress checks="EmptyBlockCheck" files="TBoundedThreadPoolServer.java"/>
|
||||||
<suppress checks="EqualsHashCode" files="StartcodeAgnosticServerName.java"/>
|
<suppress checks="EqualsHashCode" files="StartcodeAgnosticServerName.java"/>
|
||||||
|
<suppress checks="MethodLength" files="Branch1CoprocessorMethods.java"/>
|
||||||
</suppressions>
|
</suppressions>
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,6 @@ import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configurable;
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.util.Tool;
|
import org.apache.hadoop.util.Tool;
|
||||||
|
|
@ -46,9 +45,9 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
|
||||||
* command-line argument parsing.
|
* command-line argument parsing.
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public abstract class AbstractHBaseTool implements Tool, Configurable {
|
public abstract class AbstractHBaseTool implements Tool {
|
||||||
protected static final int EXIT_SUCCESS = 0;
|
public static final int EXIT_SUCCESS = 0;
|
||||||
protected static final int EXIT_FAILURE = 1;
|
public static final int EXIT_FAILURE = 1;
|
||||||
|
|
||||||
public static final String SHORT_HELP_OPTION = "h";
|
public static final String SHORT_HELP_OPTION = "h";
|
||||||
public static final String LONG_HELP_OPTION = "help";
|
public static final String LONG_HELP_OPTION = "help";
|
||||||
|
|
|
||||||
|
|
@ -21,13 +21,9 @@ package org.apache.hadoop.hbase.coprocessor;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.Coprocessor;
|
|
||||||
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
||||||
import org.apache.hadoop.hbase.TableName;
|
|
||||||
import org.apache.yetus.audience.InterfaceAudience;
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
import org.apache.yetus.audience.InterfaceStability;
|
import org.apache.yetus.audience.InterfaceStability;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest;
|
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Coprocessors implement this interface to observe and mediate bulk load operations.
|
* Coprocessors implement this interface to observe and mediate bulk load operations.
|
||||||
|
|
@ -55,7 +51,7 @@ public interface BulkLoadObserver {
|
||||||
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
|
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
|
||||||
* If you need to get the region or table name, get it from the
|
* If you need to get the region or table name, get it from the
|
||||||
* <code>ctx</code> as follows: <code>code>ctx.getEnvironment().getRegion()</code>. Use
|
* <code>ctx</code> as follows: <code>code>ctx.getEnvironment().getRegion()</code>. Use
|
||||||
* getRegionInfo to fetch the encodedName and use getTabldDescriptor() to get the tableName.
|
* getRegionInfo to fetch the encodedName and use getTableDescriptor() to get the tableName.
|
||||||
* @param ctx the environment to interact with the framework and master
|
* @param ctx the environment to interact with the framework and master
|
||||||
*/
|
*/
|
||||||
default void prePrepareBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx)
|
default void prePrepareBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx)
|
||||||
|
|
@ -66,7 +62,7 @@ public interface BulkLoadObserver {
|
||||||
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
|
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
|
||||||
* If you need to get the region or table name, get it from the
|
* If you need to get the region or table name, get it from the
|
||||||
* <code>ctx</code> as follows: <code>code>ctx.getEnvironment().getRegion()</code>. Use
|
* <code>ctx</code> as follows: <code>code>ctx.getEnvironment().getRegion()</code>. Use
|
||||||
* getRegionInfo to fetch the encodedName and use getTabldDescriptor() to get the tableName.
|
* getRegionInfo to fetch the encodedName and use getTableDescriptor() to get the tableName.
|
||||||
* @param ctx the environment to interact with the framework and master
|
* @param ctx the environment to interact with the framework and master
|
||||||
*/
|
*/
|
||||||
default void preCleanupBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx)
|
default void preCleanupBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,108 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.tool;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
||||||
|
import org.apache.hadoop.hbase.client.Admin;
|
||||||
|
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.client.Connection;
|
||||||
|
import org.apache.hadoop.hbase.client.ConnectionFactory;
|
||||||
|
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
|
||||||
|
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
|
||||||
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
|
||||||
|
|
||||||
|
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
|
||||||
|
public class DataBlockEncodingValidator extends AbstractHBaseTool {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DataBlockEncodingValidator.class);
|
||||||
|
private static final byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check DataBlockEncodings of column families are compatible.
|
||||||
|
*
|
||||||
|
* @return number of column families with incompatible DataBlockEncoding
|
||||||
|
* @throws IOException if a remote or network exception occurs
|
||||||
|
*/
|
||||||
|
private int validateDBE() throws IOException {
|
||||||
|
int incompatibilities = 0;
|
||||||
|
|
||||||
|
LOG.info("Validating Data Block Encodings");
|
||||||
|
|
||||||
|
try (Connection connection = ConnectionFactory.createConnection(getConf());
|
||||||
|
Admin admin = connection.getAdmin()) {
|
||||||
|
List<TableDescriptor> tableDescriptors = admin.listTableDescriptors();
|
||||||
|
String encoding = "";
|
||||||
|
|
||||||
|
for (TableDescriptor td : tableDescriptors) {
|
||||||
|
ColumnFamilyDescriptor[] columnFamilies = td.getColumnFamilies();
|
||||||
|
for (ColumnFamilyDescriptor cfd : columnFamilies) {
|
||||||
|
try {
|
||||||
|
encoding = Bytes.toString(cfd.getValue(DATA_BLOCK_ENCODING));
|
||||||
|
// IllegalArgumentException will be thrown if encoding is incompatible with 2.0
|
||||||
|
DataBlockEncoding.valueOf(encoding);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
incompatibilities++;
|
||||||
|
LOG.warn("Incompatible DataBlockEncoding for table: {}, cf: {}, encoding: {}",
|
||||||
|
td.getTableName().getNameAsString(), cfd.getNameAsString(), encoding);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (incompatibilities > 0) {
|
||||||
|
LOG.warn("There are {} column families with incompatible Data Block Encodings. Do not "
|
||||||
|
+ "upgrade until these encodings are converted to a supported one.", incompatibilities);
|
||||||
|
LOG.warn("Check http://hbase.apache.org/book.html#upgrade2.0.prefix-tree.removed "
|
||||||
|
+ "for instructions.");
|
||||||
|
} else {
|
||||||
|
LOG.info("The used Data Block Encodings are compatible with HBase 2.0.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return incompatibilities;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void printUsage() {
|
||||||
|
String header = "hbase " + PreUpgradeValidator.TOOL_NAME + " " +
|
||||||
|
PreUpgradeValidator.VALIDATE_DBE_NAME;
|
||||||
|
printUsage(header, null, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void addOptions() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void processOptions(CommandLine cmd) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected int doWork() throws Exception {
|
||||||
|
return (validateDBE() == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -18,111 +18,99 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.tool;
|
package org.apache.hadoop.hbase.tool;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
||||||
import org.apache.hadoop.hbase.client.Admin;
|
import org.apache.hadoop.hbase.tool.coprocessor.CoprocessorValidator;
|
||||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.client.Connection;
|
|
||||||
import org.apache.hadoop.hbase.client.ConnectionFactory;
|
|
||||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
|
|
||||||
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
|
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.util.Tool;
|
||||||
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
import org.apache.yetus.audience.InterfaceAudience;
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tool for validating that cluster can be upgraded from HBase 1.x to 2.0
|
* Tool for validating that cluster can be upgraded from HBase 1.x to 2.0
|
||||||
* <p>
|
* <p>
|
||||||
* Available validations:
|
* Available validations:
|
||||||
* <ul>
|
* <ul>
|
||||||
* <li>all: Run all pre-upgrade validations</li>
|
* <li>validate-cp: Validates Co-processors compatibility</li>
|
||||||
* <li>validateDBE: Check Data Block Encoding for column families</li>
|
* <li>validate-dbe: Check Data Block Encoding for column families</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
|
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
|
||||||
public class PreUpgradeValidator extends AbstractHBaseTool {
|
public class PreUpgradeValidator implements Tool {
|
||||||
|
private static final Logger LOG = LoggerFactory
|
||||||
|
.getLogger(PreUpgradeValidator.class);
|
||||||
|
|
||||||
public static final String NAME = "pre-upgrade";
|
public static final String TOOL_NAME = "pre-upgrade";
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(PreUpgradeValidator.class);
|
public static final String VALIDATE_CP_NAME = "validate-cp";
|
||||||
private static final byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
|
public static final String VALIDATE_DBE_NAME = "validate-dbe";
|
||||||
private boolean validateAll;
|
|
||||||
private boolean validateDBE;
|
|
||||||
|
|
||||||
/**
|
private Configuration configuration;
|
||||||
* Check DataBlockEncodings of column families are compatible.
|
|
||||||
*
|
|
||||||
* @return number of column families with incompatible DataBlockEncoding
|
|
||||||
* @throws IOException if a remote or network exception occurs
|
|
||||||
*/
|
|
||||||
private int validateDBE() throws IOException {
|
|
||||||
int incompatibilities = 0;
|
|
||||||
|
|
||||||
LOG.info("Validating Data Block Encodings");
|
@Override
|
||||||
|
public Configuration getConf() {
|
||||||
try (Connection connection = ConnectionFactory.createConnection(getConf());
|
return configuration;
|
||||||
Admin admin = connection.getAdmin()) {
|
|
||||||
List<TableDescriptor> tableDescriptors = admin.listTableDescriptors();
|
|
||||||
String encoding = "";
|
|
||||||
|
|
||||||
for (TableDescriptor td : tableDescriptors) {
|
|
||||||
ColumnFamilyDescriptor[] columnFamilies = td.getColumnFamilies();
|
|
||||||
for (ColumnFamilyDescriptor cfd : columnFamilies) {
|
|
||||||
try {
|
|
||||||
encoding = Bytes.toString(cfd.getValue(DATA_BLOCK_ENCODING));
|
|
||||||
// IllegalArgumentException will be thrown if encoding is incompatible with 2.0
|
|
||||||
DataBlockEncoding.valueOf(encoding);
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
incompatibilities++;
|
|
||||||
LOG.warn("Incompatible DataBlockEncoding for table: {}, cf: {}, encoding: {}",
|
|
||||||
td.getTableName().getNameAsString(), cfd.getNameAsString(), encoding);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (incompatibilities > 0) {
|
|
||||||
LOG.warn("There are {} column families with incompatible Data Block Encodings. Do not "
|
|
||||||
+ "upgrade until these encodings are converted to a supported one.", incompatibilities);
|
|
||||||
LOG.warn("Check http://hbase.apache.org/book.html#upgrade2.0.prefix-tree.removed "
|
|
||||||
+ "for instructions.");
|
|
||||||
} else {
|
|
||||||
LOG.info("The used Data Block Encodings are compatible with HBase 2.0.");
|
|
||||||
}
|
|
||||||
return incompatibilities;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void addOptions() {
|
public void setConf(Configuration conf) {
|
||||||
addOptNoArg("all", "Run all pre-upgrade validations");
|
this.configuration = conf;
|
||||||
addOptNoArg("validateDBE", "Validate DataBlockEncodings are compatible");
|
}
|
||||||
|
|
||||||
|
private void printUsage() {
|
||||||
|
System.out.println("usage: hbase " + TOOL_NAME + " command ...");
|
||||||
|
System.out.println("Available commands:");
|
||||||
|
System.out.printf(" %-12s Validate co-processors are compatible with HBase%n",
|
||||||
|
VALIDATE_CP_NAME);
|
||||||
|
System.out.printf(" %-12s Validate DataBlockEncoding are compatible on the cluster%n",
|
||||||
|
VALIDATE_DBE_NAME);
|
||||||
|
System.out.println("For further information, please use command -h");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void processOptions(CommandLine cmd) {
|
public int run(String[] args) throws Exception {
|
||||||
validateAll = cmd.hasOption("all");
|
if (args.length == 0) {
|
||||||
validateDBE = cmd.hasOption("validateDBE");
|
printUsage();
|
||||||
|
return AbstractHBaseTool.EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
Tool tool;
|
||||||
protected int doWork() throws Exception {
|
|
||||||
boolean validationFailed = false;
|
switch (args[0]) {
|
||||||
if (validateDBE || validateAll) {
|
case VALIDATE_CP_NAME:
|
||||||
if (validateDBE() > 0) {
|
tool = new CoprocessorValidator();
|
||||||
validationFailed = true;
|
break;
|
||||||
}
|
case VALIDATE_DBE_NAME:
|
||||||
|
tool = new DataBlockEncodingValidator();
|
||||||
|
break;
|
||||||
|
case "-h":
|
||||||
|
printUsage();
|
||||||
|
return AbstractHBaseTool.EXIT_FAILURE;
|
||||||
|
default:
|
||||||
|
System.err.println("Unknown command: " + args[0]);
|
||||||
|
printUsage();
|
||||||
|
return AbstractHBaseTool.EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
||||||
return validationFailed ? 1 : 0;
|
tool.setConf(getConf());
|
||||||
|
return tool.run(Arrays.copyOfRange(args, 1, args.length));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
new PreUpgradeValidator().doStaticMain(args);
|
int ret;
|
||||||
|
|
||||||
|
try {
|
||||||
|
ret = ToolRunner.run(HBaseConfiguration.create(), new PreUpgradeValidator(), args);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("Error running command-line tool", e);
|
||||||
|
ret = AbstractHBaseTool.EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
System.exit(ret);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,73 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hbase.tool.coprocessor;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class CoprocessorMethod {
|
||||||
|
private final String name;
|
||||||
|
private final List<String> parameters;
|
||||||
|
|
||||||
|
public CoprocessorMethod(String name) {
|
||||||
|
this.name = name;
|
||||||
|
|
||||||
|
parameters = new ArrayList<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
public CoprocessorMethod withParameters(String ... parameters) {
|
||||||
|
for (String parameter : parameters) {
|
||||||
|
this.parameters.add(parameter);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CoprocessorMethod withParameters(Class<?> ... parameters) {
|
||||||
|
for (Class<?> parameter : parameters) {
|
||||||
|
this.parameters.add(parameter.getCanonicalName());
|
||||||
|
}
|
||||||
|
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (obj == this) {
|
||||||
|
return true;
|
||||||
|
} else if (!(obj instanceof CoprocessorMethod)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
CoprocessorMethod other = (CoprocessorMethod)obj;
|
||||||
|
|
||||||
|
return Objects.equals(name, other.name) &&
|
||||||
|
Objects.equals(parameters, other.parameters);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(name, parameters);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hbase.tool.coprocessor;
|
||||||
|
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class CoprocessorMethods {
|
||||||
|
private final Set<CoprocessorMethod> methods;
|
||||||
|
|
||||||
|
public CoprocessorMethods() {
|
||||||
|
methods = new HashSet<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addMethod(String name, String ... parameters) {
|
||||||
|
CoprocessorMethod cpMethod = new CoprocessorMethod(name).withParameters(parameters);
|
||||||
|
methods.add(cpMethod);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addMethod(String name, Class<?> ... parameters) {
|
||||||
|
CoprocessorMethod cpMethod = new CoprocessorMethod(name).withParameters(parameters);
|
||||||
|
methods.add(cpMethod);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addMethod(Method method) {
|
||||||
|
CoprocessorMethod cpMethod = new CoprocessorMethod(method.getName())
|
||||||
|
.withParameters(method.getParameterTypes());
|
||||||
|
methods.add(cpMethod);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasMethod(String name, String ... parameters) {
|
||||||
|
CoprocessorMethod method = new CoprocessorMethod(name).withParameters(parameters);
|
||||||
|
return methods.contains(method);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasMethod(String name, Class<?> ... parameters) {
|
||||||
|
CoprocessorMethod method = new CoprocessorMethod(name).withParameters(parameters);
|
||||||
|
return methods.contains(method);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasMethod(Method method) {
|
||||||
|
CoprocessorMethod cpMethod = new CoprocessorMethod(method.getName())
|
||||||
|
.withParameters(method.getParameterTypes());
|
||||||
|
return methods.contains(cpMethod);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,247 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hbase.tool.coprocessor;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.net.URLClassLoader;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.security.AccessController;
|
||||||
|
import java.security.PrivilegedAction;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.jar.JarEntry;
|
||||||
|
import java.util.jar.JarFile;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.Coprocessor;
|
||||||
|
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
||||||
|
import org.apache.hadoop.hbase.tool.PreUpgradeValidator;
|
||||||
|
import org.apache.hadoop.hbase.tool.coprocessor.CoprocessorViolation.Severity;
|
||||||
|
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
|
||||||
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||||
|
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
|
||||||
|
import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
|
||||||
|
|
||||||
|
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
|
||||||
|
public class CoprocessorValidator extends AbstractHBaseTool {
|
||||||
|
private static final Logger LOG = LoggerFactory
|
||||||
|
.getLogger(CoprocessorValidator.class);
|
||||||
|
|
||||||
|
private CoprocessorMethods branch1;
|
||||||
|
private CoprocessorMethods current;
|
||||||
|
|
||||||
|
private boolean dieOnWarnings;
|
||||||
|
private boolean scan;
|
||||||
|
private List<String> args;
|
||||||
|
|
||||||
|
public CoprocessorValidator() {
|
||||||
|
branch1 = new Branch1CoprocessorMethods();
|
||||||
|
current = new CurrentCoprocessorMethods();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This classloader implementation calls {@link #resolveClass(Class)}
|
||||||
|
* method for every loaded class. It means that some extra validation will
|
||||||
|
* take place <a
|
||||||
|
* href="https://docs.oracle.com/javase/specs/jls/se8/html/jls-12.html#jls-12.3">
|
||||||
|
* according to JLS</a>.
|
||||||
|
*/
|
||||||
|
private static final class ResolverUrlClassLoader extends URLClassLoader {
|
||||||
|
private ResolverUrlClassLoader(URL[] urls) {
|
||||||
|
super(urls, ResolverUrlClassLoader.class.getClassLoader());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Class<?> loadClass(String name) throws ClassNotFoundException {
|
||||||
|
return loadClass(name, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ResolverUrlClassLoader createClassLoader(URL[] urls) {
|
||||||
|
return AccessController.doPrivileged(new PrivilegedAction<ResolverUrlClassLoader>() {
|
||||||
|
@Override
|
||||||
|
public ResolverUrlClassLoader run() {
|
||||||
|
return new ResolverUrlClassLoader(urls);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validate(ClassLoader classLoader, String className,
|
||||||
|
List<CoprocessorViolation> violations) {
|
||||||
|
LOG.debug("Validating class '{}'.", className);
|
||||||
|
|
||||||
|
try {
|
||||||
|
Class<?> clazz = classLoader.loadClass(className);
|
||||||
|
|
||||||
|
for (Method method : clazz.getDeclaredMethods()) {
|
||||||
|
LOG.trace("Validating method '{}'.", method);
|
||||||
|
|
||||||
|
if (branch1.hasMethod(method) && !current.hasMethod(method)) {
|
||||||
|
CoprocessorViolation violation = new CoprocessorViolation(Severity.WARNING,
|
||||||
|
"Method '" + method + "' was removed from new coprocessor API, "
|
||||||
|
+ "so it won't be called by HBase.");
|
||||||
|
violations.add(violation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (ClassNotFoundException e) {
|
||||||
|
CoprocessorViolation violation = new CoprocessorViolation(Severity.ERROR,
|
||||||
|
"No such class '" + className + "'.", e);
|
||||||
|
violations.add(violation);
|
||||||
|
} catch (RuntimeException | Error e) {
|
||||||
|
CoprocessorViolation violation = new CoprocessorViolation(Severity.ERROR,
|
||||||
|
"Could not validate class '" + className + "'.", e);
|
||||||
|
violations.add(violation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<CoprocessorViolation> validate(ClassLoader classLoader, List<String> classNames) {
|
||||||
|
List<CoprocessorViolation> violations = new ArrayList<>();
|
||||||
|
|
||||||
|
for (String className : classNames) {
|
||||||
|
validate(classLoader, className, violations);
|
||||||
|
}
|
||||||
|
|
||||||
|
return violations;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<CoprocessorViolation> validate(List<URL> urls, List<String> classNames)
|
||||||
|
throws IOException {
|
||||||
|
URL[] urlArray = new URL[urls.size()];
|
||||||
|
urls.toArray(urlArray);
|
||||||
|
|
||||||
|
try (ResolverUrlClassLoader classLoader = createClassLoader(urlArray)) {
|
||||||
|
return validate(classLoader, classNames);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
protected List<String> getJarClasses(Path path) throws IOException {
|
||||||
|
try (JarFile jarFile = new JarFile(path.toFile())) {
|
||||||
|
return jarFile.stream()
|
||||||
|
.map(JarEntry::getName)
|
||||||
|
.filter((name) -> name.endsWith(".class"))
|
||||||
|
.map((name) -> name.substring(0, name.length() - 6).replace('/', '.'))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
protected List<String> filterObservers(ClassLoader classLoader,
|
||||||
|
Iterable<String> classNames) throws ClassNotFoundException {
|
||||||
|
List<String> filteredClassNames = new ArrayList<>();
|
||||||
|
|
||||||
|
for (String className : classNames) {
|
||||||
|
LOG.debug("Scanning class '{}'.", className);
|
||||||
|
|
||||||
|
Class<?> clazz = classLoader.loadClass(className);
|
||||||
|
|
||||||
|
if (Coprocessor.class.isAssignableFrom(clazz)) {
|
||||||
|
LOG.debug("Found coprocessor class '{}'.", className);
|
||||||
|
filteredClassNames.add(className);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return filteredClassNames;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void printUsage() {
|
||||||
|
String header = "hbase " + PreUpgradeValidator.TOOL_NAME + " " +
|
||||||
|
PreUpgradeValidator.VALIDATE_CP_NAME + " <jar> -scan|<classes>";
|
||||||
|
printUsage(header, "Options:", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void addOptions() {
|
||||||
|
addOptNoArg("e", "Treat warnings as errors.");
|
||||||
|
addOptNoArg("scan", "Scan jar for observers.");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void processOptions(CommandLine cmd) {
|
||||||
|
scan = cmd.hasOption("scan");
|
||||||
|
dieOnWarnings = cmd.hasOption("e");
|
||||||
|
args = cmd.getArgList();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected int doWork() throws Exception {
|
||||||
|
if (args.size() < 1) {
|
||||||
|
System.err.println("Missing jar file.");
|
||||||
|
printUsage();
|
||||||
|
return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
String jar = args.get(0);
|
||||||
|
|
||||||
|
if (args.size() == 1 && !scan) {
|
||||||
|
throw new ParseException("Missing classes or -scan option.");
|
||||||
|
} else if (args.size() > 1 && scan) {
|
||||||
|
throw new ParseException("Can't use classes with -scan option.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Path jarPath = Paths.get(jar);
|
||||||
|
URL[] urls = new URL[] { jarPath.toUri().toURL() };
|
||||||
|
|
||||||
|
List<CoprocessorViolation> violations;
|
||||||
|
|
||||||
|
try (ResolverUrlClassLoader classLoader = createClassLoader(urls)) {
|
||||||
|
List<String> classNames;
|
||||||
|
|
||||||
|
if (scan) {
|
||||||
|
List<String> jarClassNames = getJarClasses(jarPath);
|
||||||
|
classNames = filterObservers(classLoader, jarClassNames);
|
||||||
|
} else {
|
||||||
|
classNames = args.subList(1, args.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
violations = validate(classLoader, classNames);
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean error = false;
|
||||||
|
|
||||||
|
for (CoprocessorViolation violation : violations) {
|
||||||
|
switch (violation.getSeverity()) {
|
||||||
|
case WARNING:
|
||||||
|
System.err.println("[WARNING] " + violation.getMessage());
|
||||||
|
|
||||||
|
if (dieOnWarnings) {
|
||||||
|
error = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
case ERROR:
|
||||||
|
System.err.println("[ERROR] " + violation.getMessage());
|
||||||
|
error = true;
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (error) ? EXIT_FAILURE : EXIT_SUCCESS;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hbase.tool.coprocessor;
|
||||||
|
|
||||||
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
|
||||||
|
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class CoprocessorViolation {
|
||||||
|
public enum Severity {
|
||||||
|
WARNING, ERROR
|
||||||
|
}
|
||||||
|
|
||||||
|
private final Severity severity;
|
||||||
|
private final String message;
|
||||||
|
|
||||||
|
public CoprocessorViolation(Severity severity, String message) {
|
||||||
|
this(severity, message, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CoprocessorViolation(Severity severity, String message, Throwable t) {
|
||||||
|
this.severity = severity;
|
||||||
|
|
||||||
|
if (t == null) {
|
||||||
|
this.message = message;
|
||||||
|
} else {
|
||||||
|
this.message = message + "\n" + Throwables.getStackTraceAsString(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Severity getSeverity() {
|
||||||
|
return severity;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getMessage() {
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hbase.tool.coprocessor;
|
||||||
|
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver;
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.EndpointObserver;
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.WALObserver;
|
||||||
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class CurrentCoprocessorMethods extends CoprocessorMethods {
|
||||||
|
public CurrentCoprocessorMethods() {
|
||||||
|
addMethods(BulkLoadObserver.class);
|
||||||
|
addMethods(EndpointObserver.class);
|
||||||
|
addMethods(MasterObserver.class);
|
||||||
|
addMethods(RegionObserver.class);
|
||||||
|
addMethods(RegionServerObserver.class);
|
||||||
|
addMethods(WALObserver.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addMethods(Class<?> clazz) {
|
||||||
|
for (Method method : clazz.getDeclaredMethods()) {
|
||||||
|
addMethod(method);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,177 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hbase.tool.coprocessor;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.Coprocessor;
|
||||||
|
import org.apache.hadoop.hbase.CoprocessorEnvironment;
|
||||||
|
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||||
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
|
||||||
|
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
|
||||||
|
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||||
|
import org.apache.hadoop.hbase.tool.coprocessor.CoprocessorViolation.Severity;
|
||||||
|
import org.junit.ClassRule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.experimental.categories.Category;
|
||||||
|
|
||||||
|
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
|
||||||
|
|
||||||
|
@Category({ SmallTests.class })
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
|
public class CoprocessorValidatorTest {
|
||||||
|
@ClassRule
|
||||||
|
public static final HBaseClassTestRule CLASS_RULE =
|
||||||
|
HBaseClassTestRule.forClass(CoprocessorValidatorTest.class);
|
||||||
|
|
||||||
|
private CoprocessorValidator validator;
|
||||||
|
|
||||||
|
public CoprocessorValidatorTest() {
|
||||||
|
validator = new CoprocessorValidator();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ClassLoader getClassLoader() {
|
||||||
|
return CoprocessorValidatorTest.class.getClassLoader();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getFullClassName(String className) {
|
||||||
|
return CoprocessorValidatorTest.class.getName() + "$" + className;
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings({"rawtypes", "unused"})
|
||||||
|
private static class TestObserver implements Coprocessor {
|
||||||
|
@Override
|
||||||
|
public void start(CoprocessorEnvironment env) throws IOException {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stop(CoprocessorEnvironment env) throws IOException {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFilterObservers() throws Exception {
|
||||||
|
String filterObservers = getFullClassName("TestObserver");
|
||||||
|
List<String> classNames = Lists.newArrayList(
|
||||||
|
filterObservers, getClass().getName());
|
||||||
|
List<String> filteredClassNames = validator.filterObservers(getClassLoader(), classNames);
|
||||||
|
|
||||||
|
assertEquals(1, filteredClassNames.size());
|
||||||
|
assertEquals(filterObservers, filteredClassNames.get(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<CoprocessorViolation> validate(String className) {
|
||||||
|
ClassLoader classLoader = getClass().getClassLoader();
|
||||||
|
return validate(classLoader, className);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<CoprocessorViolation> validate(ClassLoader classLoader, String className) {
|
||||||
|
List<String> classNames = Lists.newArrayList(getClass().getName() + "$" + className);
|
||||||
|
return validator.validate(classLoader, classNames);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* In this test case, we are try to load a not-existent class.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testNoSuchClass() throws IOException {
|
||||||
|
List<CoprocessorViolation> violations = validate("NoSuchClass");
|
||||||
|
assertEquals(1, violations.size());
|
||||||
|
|
||||||
|
CoprocessorViolation violation = violations.get(0);
|
||||||
|
assertEquals(Severity.ERROR, violation.getSeverity());
|
||||||
|
assertTrue(violation.getMessage().contains(
|
||||||
|
"java.lang.ClassNotFoundException: " +
|
||||||
|
"org.apache.hadoop.hbase.tool.coprocessor.CoprocessorValidatorTest$NoSuchClass"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* In this test case, we are validating MissingClass coprocessor, which
|
||||||
|
* references a missing class. With a special classloader, we prevent that
|
||||||
|
* class to be loaded at runtime. It simulates similar cases where a class
|
||||||
|
* is no more on our classpath.
|
||||||
|
* E.g. org.apache.hadoop.hbase.regionserver.wal.WALEdit was moved to
|
||||||
|
* org.apache.hadoop.hbase.wal, so class loading will fail on 2.0.
|
||||||
|
*/
|
||||||
|
private static class MissingClass {
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
private static class MissingClassObserver {
|
||||||
|
public void method(MissingClass missingClass) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class MissingClassClassLoader extends ClassLoader {
|
||||||
|
public MissingClassClassLoader() {
|
||||||
|
super(getClassLoader());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Class<?> loadClass(String name) throws ClassNotFoundException {
|
||||||
|
if (name.equals(getFullClassName("MissingClass"))) {
|
||||||
|
throw new ClassNotFoundException(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return super.findClass(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMissingClass() throws IOException {
|
||||||
|
MissingClassClassLoader missingClassClassLoader = new MissingClassClassLoader();
|
||||||
|
List<CoprocessorViolation> violations = validate(missingClassClassLoader,
|
||||||
|
"MissingClassObserver");
|
||||||
|
assertEquals(1, violations.size());
|
||||||
|
|
||||||
|
CoprocessorViolation violation = violations.get(0);
|
||||||
|
assertEquals(Severity.ERROR, violation.getSeverity());
|
||||||
|
assertTrue(violation.getMessage().contains(
|
||||||
|
"java.lang.ClassNotFoundException: " +
|
||||||
|
"org.apache.hadoop.hbase.tool.coprocessor.CoprocessorValidatorTest$MissingClass"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* ObsoleteMethod coprocessor implements preCreateTable method which has
|
||||||
|
* HRegionInfo parameters. In our current implementation, we pass only
|
||||||
|
* RegionInfo parameters, so this method won't be called by HBase at all.
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
private static class ObsoleteMethodObserver /* implements MasterObserver */ {
|
||||||
|
public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
|
||||||
|
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testObsoleteMethod() throws IOException {
|
||||||
|
List<CoprocessorViolation> violations = validate("ObsoleteMethodObserver");
|
||||||
|
assertEquals(1, violations.size());
|
||||||
|
|
||||||
|
CoprocessorViolation violation = violations.get(0);
|
||||||
|
assertEquals(Severity.WARNING, violation.getSeverity());
|
||||||
|
assertTrue(violation.getMessage().contains("was removed from new coprocessor API"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -845,11 +845,39 @@ The LoadTestTool has received many updates in recent HBase releases, including s
|
||||||
[[ops.pre-upgrade]]
|
[[ops.pre-upgrade]]
|
||||||
=== Pre-Upgrade validator
|
=== Pre-Upgrade validator
|
||||||
Pre-Upgrade validator tool can be used to check the cluster for known incompatibilities before upgrading from HBase 1 to HBase 2.
|
Pre-Upgrade validator tool can be used to check the cluster for known incompatibilities before upgrading from HBase 1 to HBase 2.
|
||||||
To run all the checks use the `-all` flag.
|
|
||||||
|
|
||||||
[source, bash]
|
[source, bash]
|
||||||
----
|
----
|
||||||
$ bin/hbase pre-upgrade -all
|
$ bin/hbase pre-upgrade command ...
|
||||||
|
----
|
||||||
|
|
||||||
|
==== Coprocessor validation
|
||||||
|
|
||||||
|
HBase supports co-processors for a long time, but the co-processor API can be changed between major releases. Co-processor validator tries to determine
|
||||||
|
whether the old co-processors are still compatible with the actual HBase version.
|
||||||
|
|
||||||
|
[source, bash]
|
||||||
|
----
|
||||||
|
$ bin/hbase pre-upgrade validate-cp <jar> -scan|<classes>
|
||||||
|
Options:
|
||||||
|
-e Treat warnings as errors.
|
||||||
|
-scan Scan jar for observers.
|
||||||
|
----
|
||||||
|
|
||||||
|
The first parameter of the tool is the `jar` file which holds the co-processor implementation. Further parameters can be `-scan` when the tool will
|
||||||
|
search the jar file for `Coprocessor` implementations or the `classes` can be explicitly given.
|
||||||
|
|
||||||
|
The tool can report errors and warnings. Errors mean that HBase won't be able to load the coprocessor, because it is incompatible with the current version
|
||||||
|
of HBase. Warnings mean that the co-processors can be loaded, but they won't work as expected. If `-e` option is given, then the tool will also fail
|
||||||
|
for warnings.
|
||||||
|
|
||||||
|
Please note that this tool cannot validate every aspect of jar files, it just does some static checks.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
[source, bash]
|
||||||
|
----
|
||||||
|
$ bin/hbase pre-upgrade validate-cp my-coprocessor.jar MyMasterObserver MyRegionObserver
|
||||||
----
|
----
|
||||||
|
|
||||||
==== DataBlockEncoding validation
|
==== DataBlockEncoding validation
|
||||||
|
|
@ -858,7 +886,7 @@ To verify that none of the column families are using incompatible Data Block Enc
|
||||||
|
|
||||||
[source, bash]
|
[source, bash]
|
||||||
----
|
----
|
||||||
$ bin/hbase pre-upgrade -validateDBE
|
$ bin/hbase pre-upgrade validate-dbe
|
||||||
----
|
----
|
||||||
|
|
||||||
This check validates all column families and print out any incompatibilities.
|
This check validates all column families and print out any incompatibilities.
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue