From 03f55a3e8930db35ebbe3b5f0f7cca595dd23233 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Thu, 26 May 2011 04:20:42 +0000 Subject: [PATCH] HBASE-3921 Allow adding arbitrary blobs to Put git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1127782 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../apache/hadoop/hbase/client/Delete.java | 78 ++++++++- .../org/apache/hadoop/hbase/client/Put.java | 78 ++++++++- .../hadoop/hbase/client/TestAttributes.java | 153 ++++++++++++++++++ 4 files changed, 306 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java diff --git a/CHANGES.txt b/CHANGES.txt index ac46c9936a0..b2d46a4f06b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -232,6 +232,7 @@ Release 0.91.0 - Unreleased HBASE-3841 HTable and HTableInterface docs are inconsistent with one another (Harsh J Chouraria) HBASE-2937 Facilitate Timeouts In HBase Client (Karthick Sankarachary) + HBASE-3921 Allow adding arbitrary blobs to Put (dhruba borthakur) TASKS HBASE-3559 Move report of split to master OFF the heartbeat channel diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java index 52cdc670756..a2f0c413ec2 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/Delete.java +++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java @@ -24,11 +24,14 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -66,7 +69,7 @@ import java.util.TreeMap; * timestamp. The constructor timestamp is not referenced. */ public class Delete implements Writable, Row, Comparable { - private static final byte DELETE_VERSION = (byte)1; + private static final byte DELETE_VERSION = (byte)2; private byte [] row = null; // This ts is only used when doing a deleteRow. Anything less, @@ -75,6 +78,9 @@ public class Delete implements Writable, Row, Comparable { private final Map> familyMap = new TreeMap>(Bytes.BYTES_COMPARATOR); + // a opaque blob that can be passed into a Delete. + private Map attributes; + /** Constructor for Writable. DO NOT USE */ public Delete() { this((byte [])null); @@ -283,6 +289,54 @@ public class Delete implements Writable, Row, Comparable { this.ts = timestamp; } + /** + * Sets arbitrary delete's attribute. + * In case value = null attribute is removed from the attributes map. + * @param name attribute name + * @param value attribute value + */ + public void setAttribute(String name, byte[] value) { + if (attributes == null && value == null) { + return; + } + + if (attributes == null) { + attributes = new HashMap(); + } + + if (value == null) { + attributes.remove(name); + if (attributes.isEmpty()) { + this.attributes = null; + } + } else { + attributes.put(name, value); + } + } + + /** + * Gets put's attribute + * @param name attribute name + * @return attribute value if attribute is set, null otherwise + */ + public byte[] getAttribute(String name) { + if (attributes == null) { + return null; + } + return attributes.get(name); + } + + /** + * Gets all scan's attributes + * @return unmodifiable map of all attributes + */ + public Map getAttributesMap() { + if (attributes == null) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(attributes); + } + /** * @return string */ @@ -341,6 +395,17 @@ public class Delete implements Writable, Row, Comparable { } this.familyMap.put(family, list); } + if (version > 1) { + int numAttributes = in.readInt(); + if (numAttributes > 0) { + this.attributes = new HashMap(); + for(int i=0; i { kv.write(out); } } + if (this.attributes == null) { + out.writeInt(0); + } else { + out.writeInt(this.attributes.size()); + for (Map.Entry attr : this.attributes.entrySet()) { + WritableUtils.writeString(out, attr.getKey()); + Bytes.writeByteArray(out, attr.getValue()); + } + } } /** @@ -386,6 +460,4 @@ public class Delete implements Writable, Row, Comparable { this.deleteColumn(parts[0], parts[1], HConstants.LATEST_TIMESTAMP); return this; } - - } diff --git a/src/main/java/org/apache/hadoop/hbase/client/Put.java b/src/main/java/org/apache/hadoop/hbase/client/Put.java index 0cfbfa8a73e..5a49bee0a9e 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/Put.java +++ b/src/main/java/org/apache/hadoop/hbase/client/Put.java @@ -26,12 +26,15 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -45,7 +48,7 @@ import java.util.TreeMap; * {@link #add(byte[], byte[], long, byte[]) add} if setting the timestamp. */ public class Put implements HeapSize, Writable, Row, Comparable { - private static final byte PUT_VERSION = (byte)1; + private static final byte PUT_VERSION = (byte)2; private byte [] row = null; private long timestamp = HConstants.LATEST_TIMESTAMP; @@ -55,6 +58,9 @@ public class Put implements HeapSize, Writable, Row, Comparable { private Map> familyMap = new TreeMap>(Bytes.BYTES_COMPARATOR); + // a opaque blob that can be passed into a Put. + private Map attributes; + private static final long OVERHEAD = ClassSize.align( ClassSize.OBJECT + ClassSize.REFERENCE + 2 * Bytes.SIZEOF_LONG + Bytes.SIZEOF_BOOLEAN + @@ -407,6 +413,56 @@ public class Put implements HeapSize, Writable, Row, Comparable { this.writeToWAL = write; } + /** + * Sets arbitrary put's attribute. + * In case value = null attribute is removed from the attributes map. + * @param name attribute name + * @param value attribute value + */ + public void setAttribute(String name, byte[] value) { + if (attributes == null && value == null) { + return; + } + + if (attributes == null) { + attributes = new HashMap(); + } + + if (value == null) { + attributes.remove(name); + if (attributes.isEmpty()) { + this.attributes = null; + } + } else { + attributes.put(name, value); + } + } + + /** + * Gets put's attribute + * @param name attribute name + * @return attribute value if attribute is set, null otherwise + */ + public byte[] getAttribute(String name) { + if (attributes == null) { + return null; + } + + return attributes.get(name); + } + + /** + * Gets all scan's attributes + * @return unmodifiable map of all attributes + */ + public Map getAttributesMap() { + if (attributes == null) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(attributes); + } + + /** * @return String */ @@ -502,6 +558,17 @@ public class Put implements HeapSize, Writable, Row, Comparable { } this.familyMap.put(family, keys); } + if (version > 1) { + int numAttributes = in.readInt(); + if (numAttributes > 0) { + this.attributes = new HashMap(); + for(int i=0; i { out.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); } } + if (this.attributes == null) { + out.writeInt(0); + } else { + out.writeInt(this.attributes.size()); + for (Map.Entry attr : this.attributes.entrySet()) { + WritableUtils.writeString(out, attr.getKey()); + Bytes.writeByteArray(out, attr.getValue()); + } + } } /** diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java new file mode 100644 index 00000000000..7e669564adc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java @@ -0,0 +1,153 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; + +public class TestAttributes { + @Test + public void testAttributesSerialization() throws IOException { + Put put = new Put(); + put.setAttribute("attribute1", Bytes.toBytes("value1")); + put.setAttribute("attribute2", Bytes.toBytes("value2")); + put.setAttribute("attribute3", Bytes.toBytes("value3")); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutput out = new DataOutputStream(byteArrayOutputStream); + put.write(out); + + Put put2 = new Put(); + Assert.assertTrue(put2.getAttributesMap().isEmpty()); + + put2.readFields(new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); + + Assert.assertNull(put2.getAttribute("absent")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put2.getAttribute("attribute1"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put2.getAttribute("attribute2"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), put2.getAttribute("attribute3"))); + Assert.assertEquals(3, put2.getAttributesMap().size()); + } + + @Test + public void testPutAttributes() { + Put put = new Put(); + Assert.assertTrue(put.getAttributesMap().isEmpty()); + Assert.assertNull(put.getAttribute("absent")); + + put.setAttribute("absent", null); + Assert.assertTrue(put.getAttributesMap().isEmpty()); + Assert.assertNull(put.getAttribute("absent")); + + // adding attribute + put.setAttribute("attribute1", Bytes.toBytes("value1")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttribute("attribute1"))); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttributesMap().get("attribute1"))); + + // overriding attribute value + put.setAttribute("attribute1", Bytes.toBytes("value12")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), put.getAttribute("attribute1"))); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), put.getAttributesMap().get("attribute1"))); + + // adding another attribute + put.setAttribute("attribute2", Bytes.toBytes("value2")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttribute("attribute2"))); + Assert.assertEquals(2, put.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttributesMap().get("attribute2"))); + + // removing attribute + put.setAttribute("attribute2", null); + Assert.assertNull(put.getAttribute("attribute2")); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertNull(put.getAttributesMap().get("attribute2")); + + // removing non-existed attribute + put.setAttribute("attribute2", null); + Assert.assertNull(put.getAttribute("attribute2")); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertNull(put.getAttributesMap().get("attribute2")); + + // removing another attribute + put.setAttribute("attribute1", null); + Assert.assertNull(put.getAttribute("attribute1")); + Assert.assertTrue(put.getAttributesMap().isEmpty()); + Assert.assertNull(put.getAttributesMap().get("attribute1")); + } + + + @Test + public void testDeleteAttributes() { + Delete del = new Delete(); + Assert.assertTrue(del.getAttributesMap().isEmpty()); + Assert.assertNull(del.getAttribute("absent")); + + del.setAttribute("absent", null); + Assert.assertTrue(del.getAttributesMap().isEmpty()); + Assert.assertNull(del.getAttribute("absent")); + + // adding attribute + del.setAttribute("attribute1", Bytes.toBytes("value1")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttribute("attribute1"))); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttributesMap().get("attribute1"))); + + // overriding attribute value + del.setAttribute("attribute1", Bytes.toBytes("value12")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), del.getAttribute("attribute1"))); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), del.getAttributesMap().get("attribute1"))); + + // adding another attribute + del.setAttribute("attribute2", Bytes.toBytes("value2")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttribute("attribute2"))); + Assert.assertEquals(2, del.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttributesMap().get("attribute2"))); + + // removing attribute + del.setAttribute("attribute2", null); + Assert.assertNull(del.getAttribute("attribute2")); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertNull(del.getAttributesMap().get("attribute2")); + + // removing non-existed attribute + del.setAttribute("attribute2", null); + Assert.assertNull(del.getAttribute("attribute2")); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertNull(del.getAttributesMap().get("attribute2")); + + // removing another attribute + del.setAttribute("attribute1", null); + Assert.assertNull(del.getAttribute("attribute1")); + Assert.assertTrue(del.getAttributesMap().isEmpty()); + Assert.assertNull(del.getAttributesMap().get("attribute1")); + } +}