diff --git a/CHANGES.txt b/CHANGES.txt index d196deff72b..cf3216df540 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -67,6 +67,8 @@ Release 0.18.0 - Unreleased HBASE-784 Base hbase-0.3.0 on hadoop-0.18 HBASE-841 Consolidate multiple overloaded methods in HRegionInterface, HRegionServer (Jean-Daniel Cryans via Jim Kellerman) + HBASE-840 More options on the row query in REST interface + (Sishen Freecity via Stack) NEW FEATURES HBASE-787 Postgresql to HBase table replication example (Tim Sell via Stack) diff --git a/src/java/org/apache/hadoop/hbase/HConstants.java b/src/java/org/apache/hadoop/hbase/HConstants.java index 1ff54250fbf..51c4ce6ff64 100644 --- a/src/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/java/org/apache/hadoop/hbase/HConstants.java @@ -132,7 +132,10 @@ public interface HConstants { static final byte [] ROOT_TABLE_NAME = Bytes.toBytes("-ROOT-"); /** The META table's name. */ - static final byte [] META_TABLE_NAME = Bytes.toBytes(".META."); + static final byte [] META_TABLE_NAME = Bytes.toBytes(".META."); + + /** delimiter used between portions of a region name */ + public static final int META_ROW_DELIMITER = ','; // Defines for the column names used in both ROOT and META HBase 'meta' tables. diff --git a/src/java/org/apache/hadoop/hbase/HStoreKey.java b/src/java/org/apache/hadoop/hbase/HStoreKey.java index 98f31109911..d7003c617b8 100644 --- a/src/java/org/apache/hadoop/hbase/HStoreKey.java +++ b/src/java/org/apache/hadoop/hbase/HStoreKey.java @@ -40,6 +40,12 @@ public class HStoreKey implements WritableComparable { private byte [] column = HConstants.EMPTY_BYTE_ARRAY; private long timestamp = Long.MAX_VALUE; + /* + * regionInfo is only used as a hack to compare HSKs. + * It is not serialized. See https://issues.apache.org/jira/browse/HBASE-832 + */ + private HRegionInfo regionInfo = null; + /** Default constructor used in conjunction with Writable interface */ public HStoreKey() { super(); @@ -47,8 +53,8 @@ public class HStoreKey implements WritableComparable { /** * Create an HStoreKey specifying only the row - * The column defaults to the empty string and the time stamp defaults to - * Long.MAX_VALUE + * The column defaults to the empty string, the time stamp defaults to + * Long.MAX_VALUE and the table defaults to empty string * * @param row - row key */ @@ -58,8 +64,8 @@ public class HStoreKey implements WritableComparable { /** * Create an HStoreKey specifying only the row - * The column defaults to the empty string and the time stamp defaults to - * Long.MAX_VALUE + * The column defaults to the empty string, the time stamp defaults to + * Long.MAX_VALUE and the table defaults to empty string * * @param row - row key */ @@ -69,7 +75,7 @@ public class HStoreKey implements WritableComparable { /** * Create an HStoreKey specifying the row and timestamp - * The column name defaults to the empty string + * The column and table names default to the empty string * * @param row row key * @param timestamp timestamp value @@ -80,29 +86,31 @@ public class HStoreKey implements WritableComparable { /** * Create an HStoreKey specifying the row and timestamp - * The column name defaults to the empty string + * The column and table names default to the empty string * * @param row row key * @param timestamp timestamp value */ public HStoreKey(final String row, long timestamp) { - this (row, "", timestamp); + this (row, "", timestamp, new HRegionInfo()); } /** * Create an HStoreKey specifying the row and column names * The timestamp defaults to LATEST_TIMESTAMP + * and table name defaults to the empty string * * @param row row key * @param column column key */ public HStoreKey(final String row, final String column) { - this(row, column, HConstants.LATEST_TIMESTAMP); + this(row, column, HConstants.LATEST_TIMESTAMP, new HRegionInfo()); } /** * Create an HStoreKey specifying the row and column names * The timestamp defaults to LATEST_TIMESTAMP + * and table name defaults to the empty string * * @param row row key * @param column column key @@ -110,17 +118,18 @@ public class HStoreKey implements WritableComparable { public HStoreKey(final byte [] row, final byte [] column) { this(row, column, HConstants.LATEST_TIMESTAMP); } - + /** - * Create an HStoreKey specifying all the fields - * Does not make copies of the passed byte arrays. Presumes the passed - * arrays immutable. + * Create an HStoreKey specifying the row, column names and table name + * The timestamp defaults to LATEST_TIMESTAMP + * * @param row row key * @param column column key - * @param timestamp timestamp value + * @param regionInfo region info */ - public HStoreKey(final String row, final String column, long timestamp) { - this (Bytes.toBytes(row), Bytes.toBytes(column), timestamp); + public HStoreKey(final byte [] row, + final byte [] column, final HRegionInfo regionInfo) { + this(row, column, HConstants.LATEST_TIMESTAMP, regionInfo); } /** @@ -130,12 +139,42 @@ public class HStoreKey implements WritableComparable { * @param row row key * @param column column key * @param timestamp timestamp value + * @param regionInfo region info + */ + public HStoreKey(final String row, + final String column, long timestamp, final HRegionInfo regionInfo) { + this (Bytes.toBytes(row), Bytes.toBytes(column), + timestamp, regionInfo); + } + + /** + * Create an HStoreKey specifying all the fields with unspecified table + * Does not make copies of the passed byte arrays. Presumes the passed + * arrays immutable. + * @param row row key + * @param column column key + * @param timestamp timestamp value */ public HStoreKey(final byte [] row, final byte [] column, long timestamp) { + this(row, column, timestamp, null); + } + + /** + * Create an HStoreKey specifying all the fields with specified table + * Does not make copies of the passed byte arrays. Presumes the passed + * arrays immutable. + * @param row row key + * @param column column key + * @param timestamp timestamp value + * @param regionInfo region info + */ + public HStoreKey(final byte [] row, + final byte [] column, long timestamp, final HRegionInfo regionInfo) { // Make copies this.row = row; this.column = column; this.timestamp = timestamp; + this.regionInfo = regionInfo; } /** @return Approximate size in bytes of this key. */ @@ -205,6 +244,11 @@ public class HStoreKey implements WritableComparable { return this.timestamp; } + /** @return value of regioninfo */ + public HRegionInfo getHRegionInfo() { + return this.regionInfo; + } + /** * Compares the row and column of two keys * @param other Key to compare against. Compares row and column. @@ -274,7 +318,7 @@ public class HStoreKey implements WritableComparable { /** {@inheritDoc} */ public int compareTo(Object o) { HStoreKey other = (HStoreKey)o; - int result = Bytes.compareTo(this.row, other.row); + int result = compareTwoRowKeys(this.regionInfo, this.row, other.row); if (result != 0) { return result; } @@ -419,6 +463,66 @@ public class HStoreKey implements WritableComparable { return Bytes.add(hsk.getRow(), hsk.getColumn()); } + /** + * Utility method to compare two row keys. + * This is required because of the meta delimiters. + * This is a hack. + * @param regioninfo + * @param rowA + * @param rowB + * @return value of the comparison + */ + public static int compareTwoRowKeys(HRegionInfo regionInfo, + byte[] rowA, byte[] rowB) { + if(regionInfo != null && (regionInfo.isMetaRegion() || + regionInfo.isRootRegion())) { + byte[][] keysA = stripStartKeyMeta(rowA); + byte[][] KeysB = stripStartKeyMeta(rowB); + int rowCompare = Bytes.compareTo(keysA[0], KeysB[0]); + if(rowCompare == 0) + rowCompare = Bytes.compareTo(keysA[1], KeysB[1]); + return rowCompare; + } else { + return Bytes.compareTo(rowA, rowB); + } + } + + /** + * Utility method to check if two row keys are equal. + * This is required because of the meta delimiters + * This is a hack + * @param regioninfo + * @param rowA + * @param rowB + * @return if it's equal + */ + public static boolean equalsTwoRowKeys(HRegionInfo regionInfo, + byte[] rowA, byte[] rowB) { + return rowA == null && rowB == null? true: + rowA == null && rowB != null? false: + rowA != null && rowB == null? false: + rowA.length != rowB.length? false: + compareTwoRowKeys(regionInfo,rowA,rowB) == 0; + } + + private static byte[][] stripStartKeyMeta(byte[] rowKey) { + int offset = -1; + for (int i = rowKey.length - 1; i > 0; i--) { + if (rowKey[i] == HConstants.META_ROW_DELIMITER) { + offset = i; + break; + } + } + byte [] row = new byte[offset]; + System.arraycopy(rowKey, 0, row, 0,offset); + byte [] timestamp = new byte[rowKey.length - offset - 1]; + System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1); + byte[][] elements = new byte[2][]; + elements[0] = row; + elements[1] = timestamp; + return elements; + } + // Writable /** {@inheritDoc} */ @@ -434,4 +538,4 @@ public class HStoreKey implements WritableComparable { this.column = Bytes.readByteArray(in); this.timestamp = in.readLong(); } -} \ No newline at end of file +} diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java index ed7f1df3f02..e031fb846ad 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1246,13 +1246,14 @@ public class HRegion implements HConstants { // get the closest key byte [] closestKey = store.getRowKeyAtOrBefore(row); // if it happens to be an exact match, we can stop looping - if (Bytes.equals(row, closestKey)) { + if (HStoreKey.equalsTwoRowKeys(regionInfo,row, closestKey)) { key = new HStoreKey(closestKey); break; } // otherwise, we need to check if it's the max and move to the next if (closestKey != null - && (key == null || Bytes.compareTo(closestKey, key.getRow()) > 0) ) { + && (key == null || HStoreKey.compareTwoRowKeys( + regionInfo,closestKey, key.getRow()) > 0) ) { key = new HStoreKey(closestKey); } } diff --git a/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java b/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java index 92922169e3f..302f6f76e0d 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java @@ -37,6 +37,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HStoreKey; import org.apache.hadoop.hbase.io.Cell; import org.apache.hadoop.hbase.util.Bytes; @@ -53,6 +54,8 @@ class Memcache { private final Log LOG = LogFactory.getLog(this.getClass().getName()); private final long ttl; + + private HRegionInfo regionInfo; // Note that since these structures are always accessed with a lock held, // so no additional synchronization is required. @@ -72,14 +75,17 @@ class Memcache { */ public Memcache() { this.ttl = HConstants.FOREVER; + this.regionInfo = null; } /** * Constructor. * @param ttl The TTL for cache entries, in milliseconds. + * @param regionInfo The HRI for this cache */ - public Memcache(final long ttl) { + public Memcache(final long ttl, HRegionInfo regionInfo) { this.ttl = ttl; + this.regionInfo = regionInfo; } /* @@ -383,7 +389,8 @@ class Memcache { // the search key, or a range of values between the first candidate key // and the ultimate search key (or the end of the cache) if (!tailMap.isEmpty() && - Bytes.compareTo(tailMap.firstKey().getRow(), search_key.getRow()) <= 0) { + HStoreKey.compareTwoRowKeys(regionInfo, + tailMap.firstKey().getRow(), search_key.getRow()) <= 0) { Iterator key_iterator = tailMap.keySet().iterator(); // Keep looking at cells as long as they are no greater than the @@ -391,9 +398,11 @@ class Memcache { HStoreKey deletedOrExpiredRow = null; for (HStoreKey found_key = null; key_iterator.hasNext() && (found_key == null || - Bytes.compareTo(found_key.getRow(), row) <= 0);) { + HStoreKey.compareTwoRowKeys(regionInfo, + found_key.getRow(), row) <= 0);) { found_key = key_iterator.next(); - if (Bytes.compareTo(found_key.getRow(), row) <= 0) { + if (HStoreKey.compareTwoRowKeys(regionInfo, + found_key.getRow(), row) <= 0) { if (HLogEdit.isDeleted(tailMap.get(found_key))) { HStore.handleDeleted(found_key, candidateKeys, deletes); if (deletedOrExpiredRow == null) { diff --git a/src/java/org/apache/hadoop/hbase/rest/GenericHandler.java b/src/java/org/apache/hadoop/hbase/rest/GenericHandler.java index 5025f8046ee..9173f352ed5 100644 --- a/src/java/org/apache/hadoop/hbase/rest/GenericHandler.java +++ b/src/java/org/apache/hadoop/hbase/rest/GenericHandler.java @@ -55,6 +55,7 @@ public abstract class GenericHandler { protected static final String CONTENT_TYPE = "content-type"; protected static final String ROW = "row"; protected static final String REGIONS = "regions"; + protected static final String VERSION = "version"; protected final Log LOG = LogFactory.getLog(this.getClass()); @@ -233,13 +234,32 @@ public abstract class GenericHandler { outputter.startTag(COLUMN); doElement(outputter, "name", org.apache.hadoop.hbase.util.Base64.encodeBytes(e.getKey())); - // We don't know String from binary data so we always base64 encode. - doElement(outputter, "value", - org.apache.hadoop.hbase.util.Base64.encodeBytes(e.getValue().getValue())); + outputCellXml(outputter, e.getValue()); outputter.endTag(); } } + protected void outputColumnsWithMultiVersionsXml(final XMLOutputter outputter, + final Map m) + throws IllegalStateException, IllegalArgumentException, IOException { + for (Map.Entry e: m.entrySet()) { + for (Cell c : e.getValue()) { + outputter.startTag(COLUMN); + doElement(outputter, "name", + org.apache.hadoop.hbase.util.Base64.encodeBytes(e.getKey())); + outputCellXml(outputter, c); + outputter.endTag(); + } + } + } + + protected void outputCellXml(final XMLOutputter outputter, Cell c) + throws IllegalStateException, IllegalArgumentException, IOException { + // We don't know String from binary data so we always base64 encode. + doElement(outputter, "value", + org.apache.hadoop.hbase.util.Base64.encodeBytes(c.getValue())); + doElement(outputter, "timestamp", String.valueOf(c.getTimestamp())); + } // Commented - multipart support is currently nonexistant. // protected void outputColumnsMime(final MultiPartResponse mpr, // final Map m) diff --git a/src/java/org/apache/hadoop/hbase/rest/RowHandler.java b/src/java/org/apache/hadoop/hbase/rest/RowHandler.java index 38a8c45a5b1..d8d547a03ef 100644 --- a/src/java/org/apache/hadoop/hbase/rest/RowHandler.java +++ b/src/java/org/apache/hadoop/hbase/rest/RowHandler.java @@ -2,9 +2,9 @@ package org.apache.hadoop.hbase.rest; import java.io.IOException; import java.net.URLDecoder; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; import javax.servlet.ServletException; @@ -72,7 +72,7 @@ public class RowHandler extends GenericHandler { final HttpServletResponse response, final String [] pathSegments) throws IOException { // pull the row key out of the path - String row = URLDecoder.decode(pathSegments[2], HConstants.UTF8_ENCODING); + byte[] row = Bytes.toBytes(URLDecoder.decode(pathSegments[2], HConstants.UTF8_ENCODING)); String timestampStr = null; if (pathSegments.length == 4) { @@ -85,16 +85,52 @@ public class RowHandler extends GenericHandler { } } - String[] columns = request.getParameterValues(COLUMN); - - if (columns == null || columns.length == 0) { - // They want full row returned. - - // Presumption is that this.table has already been focused on target table. - Map result = timestampStr == null ? - table.getRow(Bytes.toBytes(row)) - : table.getRow(Bytes.toBytes(row), Long.parseLong(timestampStr)); - + String[] column_params = request.getParameterValues(COLUMN); + + byte[][] columns = null; + + if (column_params != null && column_params.length > 0) { + List available_columns = new ArrayList(); + for (String column_param : column_params) { + if (column_param.length() > 0 && table.getTableDescriptor().hasFamily(Bytes.toBytes(column_param))) { + available_columns.add(column_param); + } + } + columns = Bytes.toByteArrays(available_columns.toArray(new String[0])); + } + + String[] version_params = request.getParameterValues(VERSION); + int version = 0; + if (version_params != null && version_params.length == 1) { + version = Integer.parseInt(version_params[0]); + } + + if (version > 0 && columns != null) { + Map result = new TreeMap(Bytes.BYTES_COMPARATOR); + + for (byte[] col : columns) { + Cell[] cells = timestampStr == null ? table.get(row, col, version) + : table.get(row, col, Long.parseLong(timestampStr), version); + if (cells != null) { + result.put(col, cells); + } + } + + if (result == null || result.size() == 0) { + doNotFound(response, "Row not found!"); + } else { + switch (ContentType.getContentType(request.getHeader(ACCEPT))) { + case XML: + outputRowWithMultiVersionsXml(response, result); + break; + case MIME: + default: + doNotAcceptable(response, "Unsupported Accept Header Content: " + + request.getHeader(CONTENT_TYPE)); + } + } + } else { + Map result = timestampStr == null ? table.getRow(row, columns) : table.getRow(row, columns, Long.parseLong(timestampStr)); if (result == null || result.size() == 0) { doNotFound(response, "Row not found!"); } else { @@ -104,45 +140,8 @@ public class RowHandler extends GenericHandler { break; case MIME: default: - doNotAcceptable(response, "Unsupported Accept Header Content: " + - request.getHeader(CONTENT_TYPE)); - } - } - } else { - Map prefiltered_result = table.getRow(Bytes.toBytes(row)); - - if (prefiltered_result == null || prefiltered_result.size() == 0) { - doNotFound(response, "Row not found!"); - } else { - // create a Set from the columns requested so we can - // efficiently filter the actual found columns - Set requested_columns_set = new HashSet(); - for(int i = 0; i < columns.length; i++){ - requested_columns_set.add(columns[i]); - } - - // output map that will contain the filtered results - Map m = - new TreeMap(Bytes.BYTES_COMPARATOR); - - // get an array of all the columns retrieved - Set columns_retrieved = prefiltered_result.keySet(); - - // copy over those cells with requested column names - for(byte [] current_column: columns_retrieved) { - if (requested_columns_set.contains(Bytes.toString(current_column))) { - m.put(current_column, prefiltered_result.get(current_column)); - } - } - - switch (ContentType.getContentType(request.getHeader(ACCEPT))) { - case XML: - outputRowXml(response, m); - break; - case MIME: - default: - doNotAcceptable(response, "Unsupported Accept Header Content: " + - request.getHeader(CONTENT_TYPE)); + doNotAcceptable(response, "Unsupported Accept Header Content: " + + request.getHeader(CONTENT_TYPE)); } } } @@ -167,6 +166,18 @@ public class RowHandler extends GenericHandler { outputter.getWriter().close(); } + private void outputRowWithMultiVersionsXml(final HttpServletResponse response, + final Map result) + throws IOException { + setResponseHeader(response, result.size() > 0? 200: 204, + ContentType.XML.toString()); + XMLOutputter outputter = getXMLOutputter(response.getWriter()); + outputter.startTag(ROW); + outputColumnsWithMultiVersionsXml(outputter, result); + outputter.endTag(); + outputter.endDocument(); + outputter.getWriter().close(); + } /* * @param response * @param result diff --git a/src/test/org/apache/hadoop/hbase/TestCompare.java b/src/test/org/apache/hadoop/hbase/TestCompare.java index c77ecc612b9..822a34f6549 100644 --- a/src/test/org/apache/hadoop/hbase/TestCompare.java +++ b/src/test/org/apache/hadoop/hbase/TestCompare.java @@ -53,6 +53,43 @@ public class TestCompare extends TestCase { assertTrue(nocolumn.compareTo(withcolumn) < 0); } + /** + * Tests cases where rows keys have characters below the ','. + * See HBASE-832 + */ + public void testHStoreKeyBorderCases() { + HRegionInfo info = new HRegionInfo(new HTableDescriptor("testtable"), + HConstants.EMPTY_BYTE_ARRAY,HConstants.EMPTY_BYTE_ARRAY); + HStoreKey rowA = new HStoreKey("testtable,www.hbase.org/,1234", + "", Long.MAX_VALUE, info); + HStoreKey rowB = new HStoreKey("testtable,www.hbase.org/%20,99999", + "", Long.MAX_VALUE, info); + + assertTrue(rowA.compareTo(rowB) > 0); + + rowA = new HStoreKey("testtable,www.hbase.org/,1234", + "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO); + rowB = new HStoreKey("testtable,www.hbase.org/%20,99999", + "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO); + + assertTrue(rowA.compareTo(rowB) < 0); + + rowA = new HStoreKey("testtable,,1234", + "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO); + rowB = new HStoreKey("testtable,$www.hbase.org/,99999", + "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO); + + assertTrue(rowA.compareTo(rowB) < 0); + + rowA = new HStoreKey(".META.,testtable,www.hbase.org/,1234,4321", + "", Long.MAX_VALUE, HRegionInfo.ROOT_REGIONINFO); + rowB = new HStoreKey(".META.,testtable,www.hbase.org/%20,99999,99999", + "", Long.MAX_VALUE, HRegionInfo.ROOT_REGIONINFO); + + assertTrue(rowA.compareTo(rowB) > 0); + } + + /** * Sort of HRegionInfo. */