Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-7130 NULL qualifier is ignored 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1407695 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
jxiang 2012-11-09 23:45:32 +00:00
parent f317ef5b57
commit e3b9058715
8 changed files with 130 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-server/src/main/java/org/apache/hadoop/hbase/client/Get.java
View File

@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.client;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.io.TimeRange;
@ -140,9 +140,10 @@ public class Get extends OperationWithAttributes
if(set == null) {
set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
}
if (qualifier != null) {
set.add(qualifier);
if (qualifier == null) {
qualifier = HConstants.EMPTY_BYTE_ARRAY;
}
set.add(qualifier);
familyMap.put(family, set);
return this;
}

9
hbase-server/src/main/java/org/apache/hadoop/hbase/client/Scan.java
View File

@ -21,14 +21,11 @@ package org.apache.hadoop.hbase.client;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import java.io.IOException;
import java.util.ArrayList;
@ -250,11 +247,11 @@ public class Scan extends OperationWithAttributes {
if(set == null) {
set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
}
if (qualifier != null) {
set.add(qualifier);
if (qualifier == null) {
qualifier = HConstants.EMPTY_BYTE_ARRAY;
}
set.add(qualifier);
familyMap.put(family, set);
return this;
}

47
hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java
View File

@ -79,7 +79,11 @@ public class AggregateImplementation<T, S> extends AggregateService implements
scanner = env.getRegion().getScanner(scan);
List<KeyValue> results = new ArrayList<KeyValue>();
byte[] colFamily = scan.getFamilies()[0];
byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst();
NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
byte[] qualifier = null;
if (qualifiers != null && !qualifiers.isEmpty()) {
qualifier = qualifiers.pollFirst();
}
// qualifier can be null.
boolean hasMoreRows = false;
do {
@ -129,7 +133,11 @@ public class AggregateImplementation<T, S> extends AggregateService implements
scanner = env.getRegion().getScanner(scan);
List<KeyValue> results = new ArrayList<KeyValue>();
byte[] colFamily = scan.getFamilies()[0];
byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst();
NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
byte[] qualifier = null;
if (qualifiers != null && !qualifiers.isEmpty()) {
qualifier = qualifiers.pollFirst();
}
boolean hasMoreRows = false;
do {
hasMoreRows = scanner.next(results);
@ -177,7 +185,11 @@ public class AggregateImplementation<T, S> extends AggregateService implements
Scan scan = ProtobufUtil.toScan(request.getScan());
scanner = env.getRegion().getScanner(scan);
byte[] colFamily = scan.getFamilies()[0];
byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst();
NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
byte[] qualifier = null;
if (qualifiers != null && !qualifiers.isEmpty()) {
qualifier = qualifiers.pollFirst();
}
List<KeyValue> results = new ArrayList<KeyValue>();
boolean hasMoreRows = false;
do {
@ -222,7 +234,11 @@ public class AggregateImplementation<T, S> extends AggregateService implements
try {
Scan scan = ProtobufUtil.toScan(request.getScan());
byte[] colFamily = scan.getFamilies()[0];
byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst();
NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
byte[] qualifier = null;
if (qualifiers != null && !qualifiers.isEmpty()) {
qualifier = qualifiers.pollFirst();
}
if (scan.getFilter() == null && qualifier == null)
scan.setFilter(new FirstKeyOnlyFilter());
scanner = env.getRegion().getScanner(scan);
@ -277,7 +293,11 @@ public class AggregateImplementation<T, S> extends AggregateService implements
Scan scan = ProtobufUtil.toScan(request.getScan());
scanner = env.getRegion().getScanner(scan);
byte[] colFamily = scan.getFamilies()[0];
byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst();
NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
byte[] qualifier = null;
if (qualifiers != null && !qualifiers.isEmpty()) {
qualifier = qualifiers.pollFirst();
}
List<KeyValue> results = new ArrayList<KeyValue>();
boolean hasMoreRows = false;
@ -332,7 +352,11 @@ public class AggregateImplementation<T, S> extends AggregateService implements
Scan scan = ProtobufUtil.toScan(request.getScan());
scanner = env.getRegion().getScanner(scan);
byte[] colFamily = scan.getFamilies()[0];
byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst();
NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
byte[] qualifier = null;
if (qualifiers != null && !qualifiers.isEmpty()) {
qualifier = qualifiers.pollFirst();
}
List<KeyValue> results = new ArrayList<KeyValue>();
boolean hasMoreRows = false;
@ -391,10 +415,13 @@ public class AggregateImplementation<T, S> extends AggregateService implements
Scan scan = ProtobufUtil.toScan(request.getScan());
scanner = env.getRegion().getScanner(scan);
byte[] colFamily = scan.getFamilies()[0];
NavigableSet<byte[]> quals = scan.getFamilyMap().get(colFamily);
byte[] valQualifier = quals.pollFirst();
// if weighted median is requested, get qualifier for the weight column
byte[] weightQualifier = quals.size() > 1 ? quals.pollLast() : null;
NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
byte[] valQualifier = null, weightQualifier = null;
if (qualifiers != null && !qualifiers.isEmpty()) {
valQualifier = qualifiers.pollFirst();
// if weighted median is requested, get qualifier for the weight column
weightQualifier = qualifiers.pollLast();
}
List<KeyValue> results = new ArrayList<KeyValue>();
boolean hasMoreRows = false;

38
hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
View File

@ -38,7 +38,6 @@ import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DeserializationException;
@ -643,20 +642,20 @@ public final class ProtobufUtil {
if (scan.hasFilter()) {
scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
}
Column.Builder columnBuilder = Column.newBuilder();
for (Map.Entry<byte[],NavigableSet<byte []>>
family: scan.getFamilyMap().entrySet()) {
columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
NavigableSet<byte []> columns = family.getValue();
columnBuilder.clearQualifier();
if (columns != null && columns.size() > 0) {
for (byte [] qualifier: family.getValue()) {
if (qualifier != null) {
if (scan.hasFamilies()) {
Column.Builder columnBuilder = Column.newBuilder();
for (Map.Entry<byte[],NavigableSet<byte []>>
family: scan.getFamilyMap().entrySet()) {
columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
NavigableSet<byte []> qualifiers = family.getValue();
columnBuilder.clearQualifier();
if (qualifiers != null && qualifiers.size() > 0) {
for (byte [] qualifier: qualifiers) {
columnBuilder.addQualifier(ByteString.copyFrom(qualifier));
}
}
scanBuilder.addColumn(columnBuilder.build());
}
scanBuilder.addColumn(columnBuilder.build());
}
if (scan.getMaxResultsPerColumnFamily() >= 0) {
scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
@ -723,17 +722,16 @@ public final class ProtobufUtil {
scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
}
if (proto.getColumnCount() > 0) {
TreeMap<byte [], NavigableSet<byte[]>> familyMap =
new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR);
for (Column column: proto.getColumnList()) {
byte[] family = column.getFamily().toByteArray();
TreeSet<byte []> set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
for (ByteString qualifier: column.getQualifierList()) {
set.add(qualifier.toByteArray());
if (column.getQualifierCount() > 0) {
for (ByteString qualifier: column.getQualifierList()) {
scan.addColumn(family, qualifier.toByteArray());
}
} else {
scan.addFamily(family);
}
familyMap.put(family, set);
}
scan.setFamilyMap(familyMap);
}
return scan;
}
@ -821,9 +819,7 @@ public final class ProtobufUtil {
columnBuilder.clearQualifier();
if (qualifiers != null && qualifiers.size() > 0) {
for (byte[] qualifier: qualifiers) {
if (qualifier != null) {
columnBuilder.addQualifier(ByteString.copyFrom(qualifier));
}
columnBuilder.addQualifier(ByteString.copyFrom(qualifier));
}
}
builder.addColumn(columnBuilder.build());

52
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
View File

@ -46,7 +46,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
@ -3630,6 +3629,29 @@ public class TestFromClientSide {
assertTrue(r.isEmpty());
}
@Test
public void testGet_NullQualifier() throws IOException {
HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NullQualifier"), FAMILY);
Put put = new Put(ROW);
put.add(FAMILY, QUALIFIER, VALUE);
table.put(put);
put = new Put(ROW);
put.add(FAMILY, null, VALUE);
table.put(put);
LOG.info("Row put");
Get get = new Get(ROW);
get.addColumn(FAMILY, null);
Result r = table.get(get);
assertEquals(1, r.size());
get = new Get(ROW);
get.addFamily(FAMILY);
r = table.get(get);
assertEquals(2, r.size());
}
@Test
public void testGet_NonExistentRow() throws IOException {
HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NonExistentRow"), FAMILY);
@ -4999,5 +5021,33 @@ public class TestFromClientSide {
assertEquals(1, bar.length);
}
@Test
public void testScan_NullQualifier() throws IOException {
HTable table = TEST_UTIL.createTable(Bytes.toBytes("testScan_NullQualifier"), FAMILY);
Put put = new Put(ROW);
put.add(FAMILY, QUALIFIER, VALUE);
table.put(put);
put = new Put(ROW);
put.add(FAMILY, null, VALUE);
table.put(put);
LOG.info("Row put");
Scan scan = new Scan();
scan.addColumn(FAMILY, null);
ResultScanner scanner = table.getScanner(scan);
Result[] bar = scanner.next(100);
assertEquals(1, bar.length);
assertEquals(1, bar[0].size());
scan = new Scan();
scan.addFamily(FAMILY);
scanner = table.getScanner(scan);
bar = scanner.next(100);
assertEquals(1, bar.length);
assertEquals(2, bar[0].size());
}
}

14
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
View File

@ -19,13 +19,9 @@
package org.apache.hadoop.hbase.client;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Set;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@ -102,5 +98,13 @@ public class TestGet {
Assert.assertNull(get.getAttributesMap().get("attribute1"));
}
@Test
public void testNullQualifier() {
Get get = new Get(null);
byte[] family = Bytes.toBytes("family");
get.addColumn(family, null);
Set<byte[]> qualifiers = get.getFamilyMap().get(family);
Assert.assertEquals(1, qualifiers.size());
}
}

14
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
View File

@ -19,13 +19,9 @@
package org.apache.hadoop.hbase.client;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Set;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@ -103,5 +99,13 @@ public class TestScan {
Assert.assertNull(scan.getAttributesMap().get("attribute1"));
}
@Test
public void testNullQualifier() {
Scan scan = new Scan();
byte[] family = Bytes.toBytes("family");
scan.addColumn(family, null);
Set<byte[]> qualifiers = scan.getFamilyMap().get(family);
Assert.assertEquals(1, qualifiers.size());
}
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java
View File

@ -60,7 +60,6 @@ public class TestAggregateProtocol {
private static byte[][] ROWS = makeN(ROW, ROWSIZE);
private static HBaseTestingUtility util = new HBaseTestingUtility();
private static MiniHBaseCluster cluster = null;
private static Configuration conf = util.getConfiguration();
/**
@ -75,7 +74,6 @@ public class TestAggregateProtocol {
"org.apache.hadoop.hbase.coprocessor.AggregateImplementation");
util.startMiniCluster(2);
cluster = util.getMiniHBaseCluster();
HTable table = util.createTable(TEST_TABLE, TEST_FAMILY);
util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY,
new byte[][] { HConstants.EMPTY_BYTE_ARRAY, ROWS[rowSeperator1],