Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-5228. [REST] Rip out transform feature 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1234045 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Kyle Purtell 2012-01-20 18:06:34 +00:00
parent c06edb6808
commit 7436e93ee9
7 changed files with 13 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
View File

@ -19,8 +19,6 @@
*/
package org.apache.hadoop.hbase.rest;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.rest.ResourceBase;
import org.apache.hadoop.hbase.rest.RowSpec;
@ -28,7 +26,6 @@ import org.apache.hadoop.hbase.rest.TableResource;
import org.apache.hadoop.hbase.rest.model.CellModel;
import org.apache.hadoop.hbase.rest.model.CellSetModel;
import org.apache.hadoop.hbase.rest.model.RowModel;
import org.apache.hadoop.hbase.rest.transform.Transform;
import javax.ws.rs.GET;
import javax.ws.rs.Produces;
@ -38,10 +35,8 @@ import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.util.ArrayList;
public class MultiRowResource extends ResourceBase {
private static final Log LOG = LogFactory.getLog(MultiRowResource.class);
public static final String ROW_KEYS_PARAM_NAME = "row";
TableResource tableResource;
@ -87,12 +82,9 @@ public class MultiRowResource extends ResourceBase {
KeyValue value = null;
RowModel rowModel = new RowModel(rk);
while ((value = generator.next()) != null) {
byte[] family = value.getFamily();
byte[] qualifier = value.getQualifier();
byte[] data = tableResource.transform(family, qualifier, value.getValue(), Transform.Direction.OUT);
rowModel.addCell(new CellModel(family, qualifier, value.getTimestamp(), data));
rowModel.addCell(new CellModel(value.getFamily(), value.getQualifier(),
value.getTimestamp(), value.getValue()));
}
model.addRow(rowModel);

32
src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
View File

@ -44,12 +44,10 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.rest.model.CellModel;
import org.apache.hadoop.hbase.rest.model.CellSetModel;
import org.apache.hadoop.hbase.rest.model.RowModel;
import org.apache.hadoop.hbase.rest.transform.Transform;
import org.apache.hadoop.hbase.util.Bytes;
public class RowResource extends ResourceBase {
@ -99,12 +97,8 @@ public class RowResource extends ResourceBase {
rowKey = value.getRow();
rowModel = new RowModel(rowKey);
}
byte[] family = value.getFamily();
byte[] qualifier = value.getQualifier();
byte[] data = tableResource.transform(family, qualifier,
value.getValue(), Transform.Direction.OUT);
rowModel.addCell(new CellModel(family, qualifier,
value.getTimestamp(), data));
rowModel.addCell(new CellModel(value.getFamily(), value.getQualifier(),
value.getTimestamp(), value.getValue()));
if (++count > rowspec.getMaxValues()) {
break;
}
@ -137,11 +131,7 @@ public class RowResource extends ResourceBase {
throw new WebApplicationException(Response.Status.NOT_FOUND);
}
KeyValue value = generator.next();
byte[] family = value.getFamily();
byte[] qualifier = value.getQualifier();
byte[] data = tableResource.transform(family, qualifier,
value.getValue(), Transform.Direction.OUT);
ResponseBuilder response = Response.ok(data);
ResponseBuilder response = Response.ok(value.getValue());
response.header("X-Timestamp", value.getTimestamp());
return response.build();
} catch (IOException e) {
@ -182,13 +172,9 @@ public class RowResource extends ResourceBase {
}
byte [][] parts = KeyValue.parseColumn(col);
if (parts.length == 2 && parts[1].length > 0) {
put.add(parts[0], parts[1], cell.getTimestamp(),
tableResource.transform(parts[0], parts[1], cell.getValue(),
Transform.Direction.IN));
put.add(parts[0], parts[1], cell.getTimestamp(), cell.getValue());
} else {
put.add(parts[0], null, cell.getTimestamp(),
tableResource.transform(parts[0], null, cell.getValue(),
Transform.Direction.IN));
put.add(parts[0], null, cell.getTimestamp(), cell.getValue());
}
}
puts.add(put);
@ -251,13 +237,9 @@ public class RowResource extends ResourceBase {
Put put = new Put(row);
byte parts[][] = KeyValue.parseColumn(column);
if (parts.length == 2 && parts[1].length > 0) {
put.add(parts[0], parts[1], timestamp,
tableResource.transform(parts[0], parts[1], message,
Transform.Direction.IN));
put.add(parts[0], parts[1], timestamp, message);
} else {
put.add(parts[0], null, timestamp,
tableResource.transform(parts[0], null, message,
Transform.Direction.IN));
put.add(parts[0], null, timestamp, message);
}
table = pool.getTable(tableResource.getName());
table.put(put);

196
src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
View File

@ -21,176 +21,18 @@
package org.apache.hadoop.hbase.rest;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.ws.rs.Encoded;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.rest.transform.NullTransform;
import org.apache.hadoop.hbase.rest.transform.Transform;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.StringUtils;
public class TableResource extends ResourceBase {
private static final Log LOG = LogFactory.getLog(TableResource.class);
/**
* HCD attributes starting with this string are considered transform
* directives
*/
private static final String DIRECTIVE_KEY = "Transform$";
/**
* Transform directives are of the form <tt>&lt;qualifier&gt;:&lt;class&gt;</tt>
* where <tt>qualifier</tt> is a string for exact matching or '*' as a wildcard
* that will match anything; and <tt>class</tt> is either the fully qualified
* class name of a transform implementation or can be the short name of a
* transform in the <tt>org.apache.hadoop.hbase.rest.transform package</tt>.
*/
private static final Pattern DIRECTIVE_PATTERN =
Pattern.compile("([^\\:]+)\\:([^\\,]+)\\,?");
private static final Transform defaultTransform = new NullTransform();
private static final
Map<String,Map<byte[],Map<byte[],Transform>>> transformMap =
new ConcurrentHashMap<String,Map<byte[],Map<byte[],Transform>>>();
private static final Map<String,Long> lastCheckedMap =
new ConcurrentHashMap<String,Long>();
/**
* @param table the table
* @param family the column family
* @param qualifier the column qualifier, or null
* @return the transformation specified for the given family or qualifier, if
* any, otherwise the default
*/
static Transform getTransform(String table, byte[] family, byte[] qualifier) {
if (qualifier == null) {
qualifier = HConstants.EMPTY_BYTE_ARRAY;
}
Map<byte[],Map<byte[],Transform>> familyMap = transformMap.get(table);
if (familyMap != null) {
Map<byte[],Transform> columnMap = familyMap.get(family);
if (columnMap != null) {
Transform t = columnMap.get(qualifier);
// check as necessary if there is a wildcard entry
if (t == null) {
t = columnMap.get(HConstants.EMPTY_BYTE_ARRAY);
}
// if we found something, return it, otherwise we will return the
// default by falling through
if (t != null) {
return t;
}
}
}
return defaultTransform;
}
synchronized static void setTransform(String table, byte[] family,
byte[] qualifier, Transform transform) {
Map<byte[],Map<byte[],Transform>> familyMap = transformMap.get(table);
if (familyMap == null) {
familyMap = new ConcurrentSkipListMap<byte[],Map<byte[],Transform>>(
Bytes.BYTES_COMPARATOR);
transformMap.put(table, familyMap);
}
Map<byte[],Transform> columnMap = familyMap.get(family);
if (columnMap == null) {
columnMap = new ConcurrentSkipListMap<byte[],Transform>(
Bytes.BYTES_COMPARATOR);
familyMap.put(family, columnMap);
}
// if transform is null, remove any existing entry
if (transform != null) {
columnMap.put(qualifier, transform);
} else {
columnMap.remove(qualifier);
}
}
String table;
/**
* Scan the table schema for transform directives. These are column family
* attributes containing a comma-separated list of elements of the form
* <tt>&lt;qualifier&gt;:&lt;transform-class&gt;</tt>, where qualifier
* can be a string for exact matching or '*' as a wildcard to match anything.
* The attribute key must begin with the string "Transform$".
*/
void scanTransformAttrs() throws IOException {
try {
HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
HTableDescriptor htd = admin.getTableDescriptor(Bytes.toBytes(table));
for (HColumnDescriptor hcd: htd.getFamilies()) {
for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
hcd.getValues().entrySet()) {
// does the key start with the transform directive tag?
String key = Bytes.toString(e.getKey().get());
if (!key.startsWith(DIRECTIVE_KEY)) {
// no, skip
continue;
}
// match a comma separated list of one or more directives
byte[] value = e.getValue().get();
Matcher m = DIRECTIVE_PATTERN.matcher(Bytes.toString(value));
while (m.find()) {
byte[] qualifier = HConstants.EMPTY_BYTE_ARRAY;
String s = m.group(1);
if (s.length() > 0 && !s.equals("*")) {
qualifier = Bytes.toBytes(s);
}
boolean retry = false;
String className = m.group(2);
while (true) {
try {
// if a transform was previously configured for the qualifier,
// this will simply replace it
setTransform(table, hcd.getName(), qualifier,
(Transform)Class.forName(className).newInstance());
break;
} catch (InstantiationException ex) {
LOG.error(StringUtils.stringifyException(ex));
if (retry) {
break;
}
retry = true;
} catch (IllegalAccessException ex) {
LOG.error(StringUtils.stringifyException(ex));
if (retry) {
break;
}
retry = true;
} catch (ClassNotFoundException ex) {
if (retry) {
LOG.error(StringUtils.stringifyException(ex));
break;
}
className = "org.apache.hadoop.hbase.rest.transform." + className;
retry = true;
}
}
}
}
}
} catch (TableNotFoundException e) {
// ignore
}
}
/**
* Constructor
* @param table
@ -199,22 +41,6 @@ public class TableResource extends ResourceBase {
public TableResource(String table) throws IOException {
super();
this.table = table;
// Scanning the table schema is too expensive to do for every operation.
// Do it once per minute by default.
// Setting hbase.rest.transform.check.interval to <= 0 disables rescanning.
long now = System.currentTimeMillis();
Long lastChecked = lastCheckedMap.get(table);
if (lastChecked != null) {
long interval = servlet.getConfiguration()
.getLong("hbase.rest.transform.check.interval", 60000);
if (interval > 0 && (now - lastChecked.longValue()) > interval) {
scanTransformAttrs();
lastCheckedMap.put(table, now);
}
} else {
scanTransformAttrs();
lastCheckedMap.put(table, now);
}
}
/** @return the table name */
@ -228,25 +54,11 @@ public class TableResource extends ResourceBase {
*/
boolean exists() throws IOException {
HBaseAdmin admin = new HBaseAdmin(servlet.getConfiguration());
return admin.tableExists(table);
}
/**
* Apply any configured transformations to the value
* @param family
* @param qualifier
* @param value
* @param direction
* @return
* @throws IOException
*/
byte[] transform(byte[] family, byte[] qualifier, byte[] value,
Transform.Direction direction) throws IOException {
Transform t = getTransform(table, family, qualifier);
if (t != null) {
return t.transform(value, direction);
try {
return admin.tableExists(table);
} finally {
admin.close();
}
return value;
}
@Path("exists")

35
src/main/java/org/apache/hadoop/hbase/rest/transform/Base64.java
View File

@ -1,35 +0,0 @@
/*
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.rest.transform;
public class Base64 implements Transform {
@Override
public byte[] transform(byte[] data, Direction direction) {
switch (direction) {
case IN:
return com.sun.jersey.core.util.Base64.encode(data);
case OUT:
return com.sun.jersey.core.util.Base64.decode(data);
default:
throw new RuntimeException("illegal direction");
}
}
}

28
src/main/java/org/apache/hadoop/hbase/rest/transform/NullTransform.java
View File

@ -1,28 +0,0 @@
/*
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.rest.transform;
public class NullTransform implements Transform {
@Override
public byte[] transform(byte[] data, Direction direction) {
return data;
}
}

44
src/main/java/org/apache/hadoop/hbase/rest/transform/Transform.java
View File

@ -1,44 +0,0 @@
/*
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.rest.transform;
/**
* Data transformation module
*/
public interface Transform {
/*** Transfer direction */
static enum Direction {
/** From client to server */
IN,
/** From server to client */
OUT
};
/**
* Transform data from one representation to another according to
* transfer direction.
* @param data input data
* @param direction IN or OUT
* @return the transformed data
*/
byte[] transform (byte[] data, Direction direction);
}

120
src/test/java/org/apache/hadoop/hbase/rest/TestTransform.java
View File

@ -1,120 +0,0 @@
/*
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.rest;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.rest.client.Client;
import org.apache.hadoop.hbase.rest.client.Cluster;
import org.apache.hadoop.hbase.rest.client.Response;
import org.apache.hadoop.hbase.util.Bytes;
import static org.junit.Assert.*;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(MediumTests.class)
public class TestTransform {
private static final String TABLE = "TestTransform";
private static final String CFA = "a";
private static final String CFB = "b";
private static final String COLUMN_1 = CFA + ":1";
private static final String COLUMN_2 = CFB + ":2";
private static final String ROW_1 = "testrow1";
private static final byte[] VALUE_1 = Bytes.toBytes("testvalue1");
private static final byte[] VALUE_2 = Bytes.toBytes("testvalue2");
private static final byte[] VALUE_2_BASE64 = Bytes.toBytes("dGVzdHZhbHVlMg==");
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final HBaseRESTTestingUtility REST_TEST_UTIL =
new HBaseRESTTestingUtility();
private static Client client;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster();
REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration());
client = new Client(new Cluster().add("localhost",
REST_TEST_UTIL.getServletPort()));
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
if (admin.tableExists(TABLE)) {
return;
}
HTableDescriptor htd = new HTableDescriptor(TABLE);
htd.addFamily(new HColumnDescriptor(CFA));
HColumnDescriptor cfB = new HColumnDescriptor(CFB);
cfB.setValue("Transform$1", "*:Base64");
htd.addFamily(cfB);
admin.createTable(htd);
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
REST_TEST_UTIL.shutdownServletContainer();
TEST_UTIL.shutdownMiniCluster();
}
@Test
public void testTransform() throws Exception {
String path1 = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1;
String path2 = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_2;
// store value 1
Response response = client.put(path1, Constants.MIMETYPE_BINARY, VALUE_1);
assertEquals(response.getCode(), 200);
// store value 2 (stargate should transform into base64)
response = client.put(path2, Constants.MIMETYPE_BINARY, VALUE_2);
assertEquals(response.getCode(), 200);
// get the table contents directly
HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
Get get = new Get(Bytes.toBytes(ROW_1));
get.addFamily(Bytes.toBytes(CFA));
get.addFamily(Bytes.toBytes(CFB));
Result result = table.get(get);
// value 1 should not be transformed
byte[] value = result.getValue(Bytes.toBytes(CFA), Bytes.toBytes("1"));
assertNotNull(value);
assertTrue(Bytes.equals(value, VALUE_1));
// value 2 should have been base64 encoded
value = result.getValue(Bytes.toBytes(CFB), Bytes.toBytes("2"));
assertNotNull(value);
assertTrue(Bytes.equals(value, VALUE_2_BASE64));
table.close();
// stargate should decode the transformed value back to original bytes
response = client.get(path2, Constants.MIMETYPE_BINARY);
assertEquals(response.getCode(), 200);
value = response.getBody();
assertTrue(Bytes.equals(value, VALUE_2));
}
@org.junit.Rule
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
}