HBASE-19597 Fixed Checkstyle errors in hbase-spark and enabled Checkstyle to fail on violations
This commit is contained in:
parent
c0c802f62b
commit
6007725521
|
@ -713,6 +713,22 @@
|
||||||
</execution>
|
</execution>
|
||||||
</executions>
|
</executions>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>checkstyle</id>
|
||||||
|
<phase>validate</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>check</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<failOnViolation>true</failOnViolation>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
<profiles>
|
<profiles>
|
||||||
|
|
|
@ -17,10 +17,15 @@
|
||||||
|
|
||||||
package org.apache.hadoop.hbase.spark;
|
package org.apache.hadoop.hbase.spark;
|
||||||
|
|
||||||
|
import com.google.protobuf.ByteString;
|
||||||
|
import com.google.protobuf.InvalidProtocolBufferException;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.Cell;
|
import org.apache.hadoop.hbase.Cell;
|
||||||
import org.apache.yetus.audience.InterfaceAudience;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.apache.hadoop.hbase.exceptions.DeserializationException;
|
import org.apache.hadoop.hbase.exceptions.DeserializationException;
|
||||||
import org.apache.hadoop.hbase.filter.FilterBase;
|
import org.apache.hadoop.hbase.filter.FilterBase;
|
||||||
import org.apache.hadoop.hbase.spark.datasources.BytesEncoder;
|
import org.apache.hadoop.hbase.spark.datasources.BytesEncoder;
|
||||||
|
@ -29,16 +34,12 @@ import org.apache.hadoop.hbase.spark.protobuf.generated.SparkFilterProtos;
|
||||||
import org.apache.hadoop.hbase.util.ByteStringer;
|
import org.apache.hadoop.hbase.util.ByteStringer;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.spark.sql.datasources.hbase.Field;
|
import org.apache.spark.sql.datasources.hbase.Field;
|
||||||
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import scala.collection.mutable.MutableList;
|
import scala.collection.mutable.MutableList;
|
||||||
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import com.google.protobuf.InvalidProtocolBufferException;
|
|
||||||
import com.google.protobuf.ByteString;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This filter will push down all qualifier logic given to us
|
* This filter will push down all qualifier logic given to us
|
||||||
* by SparkSQL so that we have make the filters at the region server level
|
* by SparkSQL so that we have make the filters at the region server level
|
||||||
|
@ -179,7 +180,7 @@ public class SparkSQLPushDownFilter extends FilterBase{
|
||||||
/**
|
/**
|
||||||
* @param pbBytes A pb serialized instance
|
* @param pbBytes A pb serialized instance
|
||||||
* @return An instance of SparkSQLPushDownFilter
|
* @return An instance of SparkSQLPushDownFilter
|
||||||
* @throws org.apache.hadoop.hbase.exceptions.DeserializationException
|
* @throws DeserializationException if the filter cannot be parsed from the given bytes
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
public static SparkSQLPushDownFilter parseFrom(final byte[] pbBytes)
|
public static SparkSQLPushDownFilter parseFrom(final byte[] pbBytes)
|
||||||
|
|
|
@ -16,6 +16,9 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.spark.example.hbasecontext;
|
package org.apache.hadoop.hbase.spark.example.hbasecontext;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.TableName;
|
import org.apache.hadoop.hbase.TableName;
|
||||||
|
@ -27,9 +30,6 @@ import org.apache.spark.api.java.JavaRDD;
|
||||||
import org.apache.spark.api.java.JavaSparkContext;
|
import org.apache.spark.api.java.JavaSparkContext;
|
||||||
import org.apache.spark.api.java.function.Function;
|
import org.apache.spark.api.java.function.Function;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This is a simple example of deleting records in HBase
|
* This is a simple example of deleting records in HBase
|
||||||
* with the bulkDelete function.
|
* with the bulkDelete function.
|
||||||
|
|
|
@ -16,6 +16,10 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.spark.example.hbasecontext;
|
package org.apache.hadoop.hbase.spark.example.hbasecontext;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
@ -25,23 +29,21 @@ import org.apache.hadoop.hbase.spark.JavaHBaseContext;
|
||||||
import org.apache.hadoop.hbase.spark.KeyFamilyQualifier;
|
import org.apache.hadoop.hbase.spark.KeyFamilyQualifier;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.api.java.JavaRDD;
|
import org.apache.spark.api.java.JavaRDD;
|
||||||
import org.apache.spark.api.java.JavaSparkContext;
|
import org.apache.spark.api.java.JavaSparkContext;
|
||||||
import org.apache.spark.SparkConf;
|
|
||||||
import org.apache.spark.api.java.function.Function;
|
import org.apache.spark.api.java.function.Function;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run this example using command below:
|
* Run this example using command below:
|
||||||
*
|
*
|
||||||
* SPARK_HOME/bin/spark-submit --master local[2] --class org.apache.hadoop.hbase.spark.example.hbasecontext.JavaHBaseBulkLoadExample
|
* SPARK_HOME/bin/spark-submit --master local[2]
|
||||||
|
* --class org.apache.hadoop.hbase.spark.example.hbasecontext.JavaHBaseBulkLoadExample
|
||||||
* path/to/hbase-spark.jar {path/to/output/HFiles}
|
* path/to/hbase-spark.jar {path/to/output/HFiles}
|
||||||
*
|
*
|
||||||
* This example will output put hfiles in {path/to/output/HFiles}, and user can run
|
* This example will output put hfiles in {path/to/output/HFiles}, and user can run
|
||||||
* 'hbase org.apache.hadoop.hbase.tool.LoadIncrementalHFiles' to load the HFiles into table to verify this example.
|
* 'hbase org.apache.hadoop.hbase.tool.LoadIncrementalHFiles' to load the HFiles into table to
|
||||||
|
* verify this example.
|
||||||
*/
|
*/
|
||||||
final public class JavaHBaseBulkLoadExample {
|
final public class JavaHBaseBulkLoadExample {
|
||||||
private JavaHBaseBulkLoadExample() {}
|
private JavaHBaseBulkLoadExample() {}
|
||||||
|
@ -85,17 +87,21 @@ final public class JavaHBaseBulkLoadExample {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class BulkLoadFunction implements Function<String, Pair<KeyFamilyQualifier, byte[]>> {
|
public static class BulkLoadFunction
|
||||||
|
implements Function<String, Pair<KeyFamilyQualifier, byte[]>> {
|
||||||
@Override
|
@Override
|
||||||
public Pair<KeyFamilyQualifier, byte[]> call(String v1) throws Exception {
|
public Pair<KeyFamilyQualifier, byte[]> call(String v1) throws Exception {
|
||||||
if (v1 == null)
|
if (v1 == null) {
|
||||||
return null;
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
String[] strs = v1.split(",");
|
String[] strs = v1.split(",");
|
||||||
if(strs.length != 4)
|
if(strs.length != 4) {
|
||||||
return null;
|
return null;
|
||||||
KeyFamilyQualifier kfq = new KeyFamilyQualifier(Bytes.toBytes(strs[0]), Bytes.toBytes(strs[1]),
|
}
|
||||||
Bytes.toBytes(strs[2]));
|
|
||||||
|
KeyFamilyQualifier kfq = new KeyFamilyQualifier(Bytes.toBytes(strs[0]),
|
||||||
|
Bytes.toBytes(strs[1]), Bytes.toBytes(strs[2]));
|
||||||
return new Pair(kfq, Bytes.toBytes(strs[3]));
|
return new Pair(kfq, Bytes.toBytes(strs[3]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,16 +20,16 @@ import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.Cell;
|
import org.apache.hadoop.hbase.Cell;
|
||||||
import org.apache.hadoop.hbase.CellUtil;
|
import org.apache.hadoop.hbase.CellUtil;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.TableName;
|
import org.apache.hadoop.hbase.TableName;
|
||||||
import org.apache.hadoop.hbase.client.Admin;
|
import org.apache.hadoop.hbase.client.Admin;
|
||||||
import org.apache.hadoop.hbase.client.Connection;
|
import org.apache.hadoop.hbase.client.Connection;
|
||||||
|
@ -41,12 +41,11 @@ import org.apache.hadoop.hbase.client.Result;
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
import org.apache.hadoop.hbase.client.Table;
|
||||||
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
|
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
|
||||||
import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
|
|
||||||
import org.apache.hadoop.hbase.spark.example.hbasecontext.JavaHBaseBulkDeleteExample;
|
import org.apache.hadoop.hbase.spark.example.hbasecontext.JavaHBaseBulkDeleteExample;
|
||||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||||
|
import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
import org.apache.spark.api.java.JavaRDD;
|
import org.apache.spark.api.java.JavaRDD;
|
||||||
import org.apache.spark.api.java.JavaSparkContext;
|
import org.apache.spark.api.java.JavaSparkContext;
|
||||||
|
@ -61,6 +60,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import scala.Tuple2;
|
import scala.Tuple2;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.shaded.com.google.common.io.Files;
|
import org.apache.hadoop.hbase.shaded.com.google.common.io.Files;
|
||||||
|
|
||||||
@Category({MiscTests.class, MediumTests.class})
|
@Category({MiscTests.class, MediumTests.class})
|
||||||
|
@ -317,10 +317,12 @@ public class TestJavaHBaseContext implements Serializable {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
hbaseContext.bulkLoad(rdd, TableName.valueOf(tableName), new BulkLoadFunction(), output.toUri().getPath(),
|
hbaseContext.bulkLoad(rdd, TableName.valueOf(tableName), new BulkLoadFunction(),
|
||||||
new HashMap<byte[], FamilyHFileWriteOptions>(), false, HConstants.DEFAULT_MAX_FILE_SIZE);
|
output.toUri().getPath(), new HashMap<byte[], FamilyHFileWriteOptions>(), false,
|
||||||
|
HConstants.DEFAULT_MAX_FILE_SIZE);
|
||||||
|
|
||||||
try (Connection conn = ConnectionFactory.createConnection(conf); Admin admin = conn.getAdmin()) {
|
try (Connection conn = ConnectionFactory.createConnection(conf);
|
||||||
|
Admin admin = conn.getAdmin()) {
|
||||||
Table table = conn.getTable(TableName.valueOf(tableName));
|
Table table = conn.getTable(TableName.valueOf(tableName));
|
||||||
// Do bulk load
|
// Do bulk load
|
||||||
LoadIncrementalHFiles load = new LoadIncrementalHFiles(conf);
|
LoadIncrementalHFiles load = new LoadIncrementalHFiles(conf);
|
||||||
|
@ -387,11 +389,13 @@ public class TestJavaHBaseContext implements Serializable {
|
||||||
Configuration conf = htu.getConfiguration();
|
Configuration conf = htu.getConfiguration();
|
||||||
JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
|
JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
|
||||||
|
|
||||||
hbaseContext.bulkLoadThinRows(rdd, TableName.valueOf(tableName), new BulkLoadThinRowsFunction(), output.toString(),
|
hbaseContext.bulkLoadThinRows(rdd, TableName.valueOf(tableName), new BulkLoadThinRowsFunction(),
|
||||||
new HashMap<byte[], FamilyHFileWriteOptions>(), false, HConstants.DEFAULT_MAX_FILE_SIZE);
|
output.toString(), new HashMap<byte[], FamilyHFileWriteOptions>(), false,
|
||||||
|
HConstants.DEFAULT_MAX_FILE_SIZE);
|
||||||
|
|
||||||
|
|
||||||
try (Connection conn = ConnectionFactory.createConnection(conf); Admin admin = conn.getAdmin()) {
|
try (Connection conn = ConnectionFactory.createConnection(conf);
|
||||||
|
Admin admin = conn.getAdmin()) {
|
||||||
Table table = conn.getTable(TableName.valueOf(tableName));
|
Table table = conn.getTable(TableName.valueOf(tableName));
|
||||||
// Do bulk load
|
// Do bulk load
|
||||||
LoadIncrementalHFiles load = new LoadIncrementalHFiles(conf);
|
LoadIncrementalHFiles load = new LoadIncrementalHFiles(conf);
|
||||||
|
@ -429,25 +433,31 @@ public class TestJavaHBaseContext implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
public static class BulkLoadFunction implements Function<String, Pair<KeyFamilyQualifier, byte[]>> {
|
public static class BulkLoadFunction
|
||||||
|
implements Function<String, Pair<KeyFamilyQualifier, byte[]>> {
|
||||||
@Override public Pair<KeyFamilyQualifier, byte[]> call(String v1) throws Exception {
|
@Override public Pair<KeyFamilyQualifier, byte[]> call(String v1) throws Exception {
|
||||||
if (v1 == null)
|
if (v1 == null) {
|
||||||
return null;
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
String[] strs = v1.split(",");
|
String[] strs = v1.split(",");
|
||||||
if(strs.length != 4)
|
if(strs.length != 4) {
|
||||||
return null;
|
return null;
|
||||||
KeyFamilyQualifier kfq = new KeyFamilyQualifier(Bytes.toBytes(strs[0]), Bytes.toBytes(strs[1]),
|
}
|
||||||
Bytes.toBytes(strs[2]));
|
|
||||||
|
KeyFamilyQualifier kfq = new KeyFamilyQualifier(Bytes.toBytes(strs[0]),
|
||||||
|
Bytes.toBytes(strs[1]), Bytes.toBytes(strs[2]));
|
||||||
return new Pair(kfq, Bytes.toBytes(strs[3]));
|
return new Pair(kfq, Bytes.toBytes(strs[3]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class BulkLoadThinRowsFunction implements Function<List<String>, Pair<ByteArrayWrapper, FamiliesQualifiersValues>> {
|
public static class BulkLoadThinRowsFunction
|
||||||
|
implements Function<List<String>, Pair<ByteArrayWrapper, FamiliesQualifiersValues>> {
|
||||||
@Override public Pair<ByteArrayWrapper, FamiliesQualifiersValues> call(List<String> list) throws Exception {
|
@Override public Pair<ByteArrayWrapper, FamiliesQualifiersValues> call(List<String> list) {
|
||||||
if (list == null)
|
if (list == null) {
|
||||||
return null;
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
ByteArrayWrapper rowKey = null;
|
ByteArrayWrapper rowKey = null;
|
||||||
FamiliesQualifiersValues fqv = new FamiliesQualifiersValues();
|
FamiliesQualifiersValues fqv = new FamiliesQualifiersValues();
|
||||||
for (String cell : list) {
|
for (String cell : list) {
|
||||||
|
|
Loading…
Reference in New Issue