Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-584 Names in the filter interface are confusing (Clint Morgan via Jim Kellerman) (API change for filters) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@650711 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jim Kellerman 2008-04-23 01:03:38 +00:00
parent 2bf301a4df
commit 3607d3e0db
16 changed files with 2410 additions and 2426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGES.txt
View File

@ -1,4 +1,9 @@
Hbase Change Log
INCOMPATIBLE CHANGES
HBASE-584 Names in the filter interface are confusing (Clint Morgan via
Jim Kellerman) (API change for filters)
BUG FIXES
HBASE-574 HBase does not load hadoop native libs (Rong-En Fan via Stack)
HBASE-573 HBase does not read hadoop-*.xml for dfs configuration after

122
src/java/org/apache/hadoop/hbase/filter/InclusiveStopRowFilter.java
View File

@ -1,61 +1,61 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import org.apache.hadoop.io.Text;
/**
* Subclass of StopRowFilter that filters rows > the stop row,
* making it include up to the last row but no further.
*/
public class InclusiveStopRowFilter extends StopRowFilter{
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public InclusiveStopRowFilter() {super();}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public InclusiveStopRowFilter(final Text stopRowKey) {
super(stopRowKey);
}
/** {@inheritDoc} */
@Override
public boolean filter(final Text rowKey) {
if (rowKey == null) {
if (this.stopRowKey == null) {
return true;
}
return false;
}
boolean result = this.stopRowKey.compareTo(rowKey) < 0;
if (LOG.isDebugEnabled()) {
LOG.debug("Filter result for rowKey: " + rowKey + ". Result: " +
result);
}
return result;
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import org.apache.hadoop.io.Text;
/**
* Subclass of StopRowFilter that filters rows > the stop row,
* making it include up to the last row but no further.
*/
public class InclusiveStopRowFilter extends StopRowFilter{
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public InclusiveStopRowFilter() {super();}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public InclusiveStopRowFilter(final Text stopRowKey) {
super(stopRowKey);
}
/** {@inheritDoc} */
@Override
public boolean filterRowKey(final Text rowKey) {
if (rowKey == null) {
if (this.stopRowKey == null) {
return true;
}
return false;
}
boolean result = this.stopRowKey.compareTo(rowKey) < 0;
if (LOG.isDebugEnabled()) {
LOG.debug("Filter result for rowKey: " + rowKey + ". Result: " +
result);
}
return result;
}
}

316
src/java/org/apache/hadoop/hbase/filter/PageRowFilter.java
View File

@ -1,159 +1,159 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
/**
* Implementation of RowFilterInterface that limits results to a specific page
* size. It terminates scanning once the number of filter-passed results is >=
* the given page size.
*
* <p>
* Note that this filter cannot guarantee that the number of results returned
* to a client are <= page size. This is because the filter is applied
* separately on different region servers. It does however optimize the scan of
* individual HRegions by making sure that the page size is never exceeded
* locally.
* </p>
*/
public class PageRowFilter implements RowFilterInterface {
private long pageSize = Long.MAX_VALUE;
private int rowsAccepted = 0;
static final Log LOG = LogFactory.getLog(PageRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public PageRowFilter() {
super();
}
/**
* Constructor that takes a maximum page size.
*
* @param pageSize Maximum result size.
*/
public PageRowFilter(final long pageSize) {
this.pageSize = pageSize;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final Text[] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
rowsAccepted = 0;
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered,
@SuppressWarnings("unused") Text rowKey) {
if (!filtered) {
this.rowsAccepted++;
if (LOG.isDebugEnabled()) {
LOG.debug("rowProcessed incremented rowsAccepted to " +
this.rowsAccepted);
}
}
}
/**
*
* {@inheritDoc}
*/
public boolean processAlways() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
boolean result = this.rowsAccepted > this.pageSize;
if (LOG.isDebugEnabled()) {
LOG.debug("filtering decision is " + result + " with rowsAccepted: " +
this.rowsAccepted);
}
return result;
}
/**
*
* {@inheritDoc}
*/
public boolean filter(@SuppressWarnings("unused") final Text rowKey) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filter(@SuppressWarnings("unused") final Text rowKey,
@SuppressWarnings("unused") final Text colKey,
@SuppressWarnings("unused") final byte[] data) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filterNotNull(@SuppressWarnings("unused")
final SortedMap<Text, byte[]> columns) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
this.pageSize = in.readLong();
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
out.writeLong(pageSize);
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
/**
* Implementation of RowFilterInterface that limits results to a specific page
* size. It terminates scanning once the number of filter-passed results is >=
* the given page size.
*
* <p>
* Note that this filter cannot guarantee that the number of results returned
* to a client are <= page size. This is because the filter is applied
* separately on different region servers. It does however optimize the scan of
* individual HRegions by making sure that the page size is never exceeded
* locally.
* </p>
*/
public class PageRowFilter implements RowFilterInterface {
private long pageSize = Long.MAX_VALUE;
private int rowsAccepted = 0;
static final Log LOG = LogFactory.getLog(PageRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public PageRowFilter() {
super();
}
/**
* Constructor that takes a maximum page size.
*
* @param pageSize Maximum result size.
*/
public PageRowFilter(final long pageSize) {
this.pageSize = pageSize;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final Text[] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
rowsAccepted = 0;
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered,
@SuppressWarnings("unused") Text rowKey) {
if (!filtered) {
this.rowsAccepted++;
if (LOG.isDebugEnabled()) {
LOG.debug("rowProcessed incremented rowsAccepted to " +
this.rowsAccepted);
}
}
}
/**
*
* {@inheritDoc}
*/
public boolean processAlways() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
boolean result = this.rowsAccepted > this.pageSize;
if (LOG.isDebugEnabled()) {
LOG.debug("filtering decision is " + result + " with rowsAccepted: " +
this.rowsAccepted);
}
return result;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRowKey(@SuppressWarnings("unused") final Text rowKey) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filterColumn(@SuppressWarnings("unused") final Text rowKey,
@SuppressWarnings("unused") final Text colKey,
@SuppressWarnings("unused") final byte[] data) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filterRow(@SuppressWarnings("unused")
final SortedMap<Text, byte[]> columns) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
this.pageSize = in.readLong();
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
out.writeLong(pageSize);
}
}

678
src/java/org/apache/hadoop/hbase/filter/RegExpRowFilter.java
View File

@ -1,340 +1,340 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
/**
* Implementation of RowFilterInterface that can filter by rowkey regular
* expression and/or individual column values (equals comparison only).
* Multiple column filters imply an implicit conjunction of filter criteria.
*/
public class RegExpRowFilter implements RowFilterInterface {
private Pattern rowKeyPattern = null;
private String rowKeyRegExp = null;
private Map<Text, byte[]> equalsMap = new HashMap<Text, byte[]>();
private Set<Text> nullColumns = new HashSet<Text>();
static final Log LOG = LogFactory.getLog(RegExpRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RegExpRowFilter() {
super();
}
/**
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
*/
public RegExpRowFilter(final String rowKeyRegExp) {
this.rowKeyRegExp = rowKeyRegExp;
}
/**
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
* @param columnFilter
*/
public RegExpRowFilter(final String rowKeyRegExp,
final Map<Text, byte[]> columnFilter) {
this.rowKeyRegExp = rowKeyRegExp;
this.setColumnFilters(columnFilter);
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, Text rowKey) {
//doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/**
* Specify a value that must be matched for the given column.
*
* @param colKey
* the column to match on
* @param value
* the value that must equal the stored value.
*/
public void setColumnFilter(final Text colKey, final byte[] value) {
if (value == null) {
nullColumns.add(colKey);
} else {
equalsMap.put(colKey, value);
}
}
/**
* Set column filters for a number of columns.
*
* @param columnFilter
* Map of columns with value criteria.
*/
public void setColumnFilters(final Map<Text, byte[]> columnFilter) {
if (null == columnFilter) {
nullColumns.clear();
equalsMap.clear();
} else {
for (Entry<Text, byte[]> entry : columnFilter.entrySet()) {
setColumnFilter(entry.getKey(), entry.getValue());
}
}
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filter(final Text rowKey) {
if (filtersByRowKey() && rowKey != null) {
boolean result = !getRowKeyPattern().matcher(rowKey.toString()).matches();
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning " + result + " for rowKey: " + rowKey);
}
return result;
}
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filter(final Text rowKey, final Text colKey,
final byte[] data) {
if (filter(rowKey)) {
return true;
}
if (filtersByColumnValue()) {
byte[] filterValue = equalsMap.get(colKey);
if (null != filterValue) {
boolean result = !Arrays.equals(filterValue, data);
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning " + result + " for rowKey: " + rowKey +
" colKey: " + colKey);
}
return result;
}
}
if (nullColumns.contains(colKey)) {
if (data != null && !HLogEdit.isDeleted(data)) {
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning true for rowKey: " + rowKey +
" colKey: " + colKey);
}
return true;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning false for rowKey: " + rowKey + " colKey: " +
colKey);
}
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterNotNull(final SortedMap<Text, byte[]> columns) {
for (Entry<Text, byte[]> col : columns.entrySet()) {
if (nullColumns.contains(col.getKey())
&& !HLogEdit.isDeleted(col.getValue())) {
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning true for colKey: " + col.getKey()
+ ", column should be null.");
}
return true;
}
}
for (Text col : equalsMap.keySet()) {
if (!columns.containsKey(col)) {
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning true for colKey: " + col +
", column not found in given SortedMap<Text, byte[]>.");
}
return true;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning false.");
}
return false;
}
private boolean filtersByColumnValue() {
return equalsMap != null && equalsMap.size() > 0;
}
private boolean filtersByRowKey() {
return null != rowKeyPattern || null != rowKeyRegExp;
}
private String getRowKeyRegExp() {
if (null == rowKeyRegExp && rowKeyPattern != null) {
rowKeyRegExp = rowKeyPattern.toString();
}
return rowKeyRegExp;
}
private Pattern getRowKeyPattern() {
if (rowKeyPattern == null && rowKeyRegExp != null) {
rowKeyPattern = Pattern.compile(rowKeyRegExp);
}
return rowKeyPattern;
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
boolean hasRowKeyPattern = in.readBoolean();
if (hasRowKeyPattern) {
rowKeyRegExp = in.readUTF();
}
// equals map
equalsMap.clear();
int size = in.readInt();
for (int i = 0; i < size; i++) {
Text key = new Text();
key.readFields(in);
int len = in.readInt();
byte[] value = null;
if (len >= 0) {
value = new byte[len];
in.readFully(value);
}
setColumnFilter(key, value);
}
// nullColumns
nullColumns.clear();
size = in.readInt();
for (int i = 0; i < size; i++) {
Text key = new Text();
key.readFields(in);
setColumnFilter(key, null);
}
}
/**
*
* {@inheritDoc}
*/
public void validate(final Text[] columns) {
Set<Text> invalids = new HashSet<Text>();
for (Text colKey : getFilterColumns()) {
boolean found = false;
for (Text col : columns) {
if (col.equals(colKey)) {
found = true;
break;
}
}
if (!found) {
invalids.add(colKey);
}
}
if (invalids.size() > 0) {
throw new InvalidRowFilterException(String.format(
"RowFilter contains criteria on columns %s not in %s", invalids,
Arrays.toString(columns)));
}
}
private Set<Text> getFilterColumns() {
Set<Text> cols = new HashSet<Text>();
cols.addAll(equalsMap.keySet());
cols.addAll(nullColumns);
return cols;
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
if (!filtersByRowKey()) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeUTF(getRowKeyRegExp());
}
// equalsMap
out.writeInt(equalsMap.size());
for (Entry<Text, byte[]> entry : equalsMap.entrySet()) {
entry.getKey().write(out);
byte[] value = entry.getValue();
out.writeInt(value.length);
out.write(value);
}
// null columns
out.writeInt(nullColumns.size());
for (Text col : nullColumns) {
col.write(out);
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
/**
* Implementation of RowFilterInterface that can filter by rowkey regular
* expression and/or individual column values (equals comparison only).
* Multiple column filters imply an implicit conjunction of filter criteria.
*/
public class RegExpRowFilter implements RowFilterInterface {
private Pattern rowKeyPattern = null;
private String rowKeyRegExp = null;
private Map<Text, byte[]> equalsMap = new HashMap<Text, byte[]>();
private Set<Text> nullColumns = new HashSet<Text>();
static final Log LOG = LogFactory.getLog(RegExpRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RegExpRowFilter() {
super();
}
/**
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
*/
public RegExpRowFilter(final String rowKeyRegExp) {
this.rowKeyRegExp = rowKeyRegExp;
}
/**
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
* @param columnFilter
*/
public RegExpRowFilter(final String rowKeyRegExp,
final Map<Text, byte[]> columnFilter) {
this.rowKeyRegExp = rowKeyRegExp;
this.setColumnFilters(columnFilter);
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, Text rowKey) {
//doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/**
* Specify a value that must be matched for the given column.
*
* @param colKey
* the column to match on
* @param value
* the value that must equal the stored value.
*/
public void setColumnFilter(final Text colKey, final byte[] value) {
if (value == null) {
nullColumns.add(colKey);
} else {
equalsMap.put(colKey, value);
}
}
/**
* Set column filters for a number of columns.
*
* @param columnFilter
* Map of columns with value criteria.
*/
public void setColumnFilters(final Map<Text, byte[]> columnFilter) {
if (null == columnFilter) {
nullColumns.clear();
equalsMap.clear();
} else {
for (Entry<Text, byte[]> entry : columnFilter.entrySet()) {
setColumnFilter(entry.getKey(), entry.getValue());
}
}
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRowKey(final Text rowKey) {
if (filtersByRowKey() && rowKey != null) {
boolean result = !getRowKeyPattern().matcher(rowKey.toString()).matches();
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning " + result + " for rowKey: " + rowKey);
}
return result;
}
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterColumn(final Text rowKey, final Text colKey,
final byte[] data) {
if (filterRowKey(rowKey)) {
return true;
}
if (filtersByColumnValue()) {
byte[] filterValue = equalsMap.get(colKey);
if (null != filterValue) {
boolean result = !Arrays.equals(filterValue, data);
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning " + result + " for rowKey: " + rowKey +
" colKey: " + colKey);
}
return result;
}
}
if (nullColumns.contains(colKey)) {
if (data != null && !HLogEdit.isDeleted(data)) {
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning true for rowKey: " + rowKey +
" colKey: " + colKey);
}
return true;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filter returning false for rowKey: " + rowKey + " colKey: " +
colKey);
}
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRow(final SortedMap<Text, byte[]> columns) {
for (Entry<Text, byte[]> col : columns.entrySet()) {
if (nullColumns.contains(col.getKey())
&& !HLogEdit.isDeleted(col.getValue())) {
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning true for colKey: " + col.getKey()
+ ", column should be null.");
}
return true;
}
}
for (Text col : equalsMap.keySet()) {
if (!columns.containsKey(col)) {
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning true for colKey: " + col +
", column not found in given SortedMap<Text, byte[]>.");
}
return true;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning false.");
}
return false;
}
private boolean filtersByColumnValue() {
return equalsMap != null && equalsMap.size() > 0;
}
private boolean filtersByRowKey() {
return null != rowKeyPattern || null != rowKeyRegExp;
}
private String getRowKeyRegExp() {
if (null == rowKeyRegExp && rowKeyPattern != null) {
rowKeyRegExp = rowKeyPattern.toString();
}
return rowKeyRegExp;
}
private Pattern getRowKeyPattern() {
if (rowKeyPattern == null && rowKeyRegExp != null) {
rowKeyPattern = Pattern.compile(rowKeyRegExp);
}
return rowKeyPattern;
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
boolean hasRowKeyPattern = in.readBoolean();
if (hasRowKeyPattern) {
rowKeyRegExp = in.readUTF();
}
// equals map
equalsMap.clear();
int size = in.readInt();
for (int i = 0; i < size; i++) {
Text key = new Text();
key.readFields(in);
int len = in.readInt();
byte[] value = null;
if (len >= 0) {
value = new byte[len];
in.readFully(value);
}
setColumnFilter(key, value);
}
// nullColumns
nullColumns.clear();
size = in.readInt();
for (int i = 0; i < size; i++) {
Text key = new Text();
key.readFields(in);
setColumnFilter(key, null);
}
}
/**
*
* {@inheritDoc}
*/
public void validate(final Text[] columns) {
Set<Text> invalids = new HashSet<Text>();
for (Text colKey : getFilterColumns()) {
boolean found = false;
for (Text col : columns) {
if (col.equals(colKey)) {
found = true;
break;
}
}
if (!found) {
invalids.add(colKey);
}
}
if (invalids.size() > 0) {
throw new InvalidRowFilterException(String.format(
"RowFilter contains criteria on columns %s not in %s", invalids,
Arrays.toString(columns)));
}
}
private Set<Text> getFilterColumns() {
Set<Text> cols = new HashSet<Text>();
cols.addAll(equalsMap.keySet());
cols.addAll(nullColumns);
return cols;
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
if (!filtersByRowKey()) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeUTF(getRowKeyRegExp());
}
// equalsMap
out.writeInt(equalsMap.size());
for (Entry<Text, byte[]> entry : equalsMap.entrySet()) {
entry.getKey().write(out);
byte[] value = entry.getValue();
out.writeInt(value.length);
out.write(value);
}
// null columns
out.writeInt(nullColumns.size());
for (Text col : nullColumns) {
col.write(out);
}
}
}

238
src/java/org/apache/hadoop/hbase/filter/RowFilterInterface.java
View File

@ -1,125 +1,115 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.util.SortedMap;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
/**
*
* Interface used for row-level filters applied to HRegion.HScanner scan
* results during calls to next().
*/
public interface RowFilterInterface extends Writable {
/**
* Resets the state of the filter. Used prior to the start of a Region scan.
*
*/
void reset();
/**
* Called to let filter know the final decision (to pass or filter) on a
* given row. With out HScanner calling this, the filter does not know if a
* row passed filtering even if it passed the row itself because other
* filters may have failed the row. E.g. when this filter is a member of a
* RowFilterSet with an OR operator.
*
* @see RowFilterSet
* @param filtered
* @param key
*/
void rowProcessed(boolean filtered, Text key);
/**
* Returns whether or not the filter should always be processed in any
* filtering call. This precaution is necessary for filters that maintain
* state and need to be updated according to their response to filtering
* calls (see WhileMatchRowFilter for an example). At times, filters nested
* in RowFilterSets may or may not be called because the RowFilterSet
* determines a result as fast as possible. Returning true for
* processAlways() ensures that the filter will always be called.
*
* @return whether or not to always process the filter
*/
boolean processAlways();
/**
* Determines if the filter has decided that all remaining results should be
* filtered (skipped). This is used to prevent the scanner from scanning a
* the rest of the HRegion when for sure the filter will exclude all
* remaining rows.
*
* @return true if the filter intends to filter all remaining rows.
*/
boolean filterAllRemaining();
/**
* Filters on just a row key.
*
* @param rowKey
* @return true if given row key is filtered and row should not be processed.
*/
boolean filter(final Text rowKey);
/**
* Filters on row key and/or a column key.
*
* @param rowKey
* row key to filter on. May be null for no filtering of row key.
* @param colKey
* column whose data will be filtered
* @param data
* column value
* @return true if row filtered and should not be processed.
*/
boolean filter(final Text rowKey, final Text colKey, final byte[] data);
/**
* Filters a row if:
* 1) The given row (@param columns) has a columnKey expected to be null AND
* the value associated with that columnKey is non-null.
* 2) The filter has a criterion for a particular columnKey, but that
* columnKey is not in the given row (@param columns).
*
* Note that filterNotNull does not care whether the values associated with a
* columnKey match. Also note that a "null value" associated with a columnKey
* is expressed as HConstants.DELETE_BYTES.
*
* @param columns
* @return true if null/non-null criteria not met.
*/
boolean filterNotNull(final SortedMap<Text, byte[]> columns);
/**
* Validates that this filter applies only to a subset of the given columns.
* This check is done prior to opening of scanner due to the limitation that
* filtering of columns is dependent on the retrieval of those columns within
* the HRegion. Criteria on columns that are not part of a scanner's column
* list will be ignored. In the case of null value filters, all rows will pass
* the filter. This behavior should be 'undefined' for the user and therefore
* not permitted.
*
* @param columns
*/
void validate(final Text[] columns);
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.util.SortedMap;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
/**
*
* Interface used for row-level filters applied to HRegion.HScanner scan
* results during calls to next().
*/
public interface RowFilterInterface extends Writable {
/**
* Resets the state of the filter. Used prior to the start of a Region scan.
*
*/
void reset();
/**
* Called to let filter know the final decision (to pass or filter) on a
* given row. With out HScanner calling this, the filter does not know if a
* row passed filtering even if it passed the row itself because other
* filters may have failed the row. E.g. when this filter is a member of a
* RowFilterSet with an OR operator.
*
* @see RowFilterSet
* @param filtered
* @param key
*/
void rowProcessed(boolean filtered, Text key);
/**
* Returns whether or not the filter should always be processed in any
* filtering call. This precaution is necessary for filters that maintain
* state and need to be updated according to their response to filtering
* calls (see WhileMatchRowFilter for an example). At times, filters nested
* in RowFilterSets may or may not be called because the RowFilterSet
* determines a result as fast as possible. Returning true for
* processAlways() ensures that the filter will always be called.
*
* @return whether or not to always process the filter
*/
boolean processAlways();
/**
* Determines if the filter has decided that all remaining results should be
* filtered (skipped). This is used to prevent the scanner from scanning a
* the rest of the HRegion when for sure the filter will exclude all
* remaining rows.
*
* @return true if the filter intends to filter all remaining rows.
*/
boolean filterAllRemaining();
/**
* Filters on just a row key. This is the first chance to stop a row.
*
* @param rowKey
* @return true if given row key is filtered and row should not be processed.
*/
boolean filterRowKey(final Text rowKey);
/**
* Filters on row key, column name, and column value. This will take individual columns out of a row,
* but the rest of the row will still get through.
*
* @param rowKey row key to filter on.
* @param colunmName column name to filter on
* @param columnValue column value to filter on
* @return true if row filtered and should not be processed.
*/
boolean filterColumn(final Text rowKey, final Text colunmName, final byte[] columnValue);
/**
* Filter on the fully assembled row. This is the last chance to stop a row.
*
* @param columns
* @return true if row filtered and should not be processed.
*/
boolean filterRow(final SortedMap<Text, byte[]> columns);
/**
* Validates that this filter applies only to a subset of the given columns.
* This check is done prior to opening of scanner due to the limitation that
* filtering of columns is dependent on the retrieval of those columns within
* the HRegion. Criteria on columns that are not part of a scanner's column
* list will be ignored. In the case of null value filters, all rows will pass
* the filter. This behavior should be 'undefined' for the user and therefore
* not permitted.
*
* @param columns
*/
void validate(final Text[] columns);
}

615
src/java/org/apache/hadoop/hbase/filter/RowFilterSet.java
View File

@ -1,313 +1,302 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
/**
* Implementation of RowFilterInterface that represents a set of RowFilters
* which will be evaluated with a specified boolean operator MUST_PASS_ALL
* (!AND) or MUST_PASS_ONE (!OR). Since you can use RowFilterSets as children
* of RowFilterSet, you can create a hierarchy of filters to be evaluated.
*/
public class RowFilterSet implements RowFilterInterface {
/** set operator */
public static enum Operator {
/** !AND */
MUST_PASS_ALL,
/** !OR */
MUST_PASS_ONE
}
private Operator operator = Operator.MUST_PASS_ALL;
private Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
static final Log LOG = LogFactory.getLog(RowFilterSet.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RowFilterSet() {
super();
}
/**
* Constructor that takes a set of RowFilters. The default operator
* MUST_PASS_ALL is assumed.
*
* @param rowFilters
*/
public RowFilterSet(final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
}
/**
* Constructor that takes a set of RowFilters and an operator.
*
* @param operator Operator to process filter set with.
* @param rowFilters Set of row filters.
*/
public RowFilterSet(final Operator operator,
final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
this.operator = operator;
}
/** {@inheritDoc} */
public void validate(final Text[] columns) {
for (RowFilterInterface filter : filters) {
filter.validate(columns);
if (LOG.isDebugEnabled()) {
LOG.debug("Validated subfilter of type " +
filter.getClass().getSimpleName());
}
}
}
/** {@inheritDoc} */
public void reset() {
for (RowFilterInterface filter : filters) {
filter.reset();
if (LOG.isDebugEnabled()) {
LOG.debug("Reset subfilter of type " +
filter.getClass().getSimpleName());
}
}
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, Text rowKey) {
for (RowFilterInterface filter : filters) {
filter.rowProcessed(filtered, rowKey);
if (LOG.isDebugEnabled()) {
LOG.debug("Called rowProcessed on subfilter of type " +
filter.getClass().getSimpleName());
}
}
}
/** {@inheritDoc} */
public boolean processAlways() {
for (RowFilterInterface filter : filters) {
if (filter.processAlways()) {
if (LOG.isDebugEnabled()) {
LOG.debug("processAlways() is true due to subfilter of type " +
filter.getClass().getSimpleName());
}
return true;
}
}
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining()) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filterAllRemaining returning true due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
return true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining()) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filterAllRemaining returning false due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
return false;
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filterAllRemaining default returning " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filter(final Text rowKey) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filter(rowKey)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filter(Text) will return true due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filter(rowKey)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filter(Text) will return false due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filter(rowKey);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filter(Text) returning " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filter(final Text rowKey, final Text colKey,
final byte[] data) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() ||
filter.filter(rowKey, colKey, data)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filter(Text, Text, byte[]) will" +
" return true due to subfilter of type " +
filter.getClass().getSimpleName());
}
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() &&
!filter.filter(rowKey, colKey, data)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filter(Text, Text, byte[]) will" +
" return false due to subfilter of type " +
filter.getClass().getSimpleName());
}
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filter(rowKey, colKey, data);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filter(Text, Text, byte[]) returning " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filterNotNull(final SortedMap<Text, byte[]> columns) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterNotNull(columns)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filterNotNull will return true due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filterNotNull(columns)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filterNotNull will return false due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterNotNull(columns);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning " + result);
}
return result;
}
/** {@inheritDoc} */
public void readFields(final DataInput in) throws IOException {
byte opByte = in.readByte();
operator = Operator.values()[opByte];
int size = in.readInt();
if (size > 0) {
filters = new HashSet<RowFilterInterface>();
try {
for (int i = 0; i < size; i++) {
String className = in.readUTF();
Class<?> clazz = Class.forName(className);
RowFilterInterface filter;
filter = (RowFilterInterface) clazz.newInstance();
filter.readFields(in);
filters.add(filter);
if (LOG.isDebugEnabled()) {
LOG.debug("Successfully read in subfilter of type " +
filter.getClass().getSimpleName());
}
}
} catch (InstantiationException e) {
throw new RuntimeException("Failed to deserialize RowFilterInterface.",
e);
} catch (IllegalAccessException e) {
throw new RuntimeException("Failed to deserialize RowFilterInterface.",
e);
} catch (ClassNotFoundException e) {
throw new RuntimeException("Failed to deserialize RowFilterInterface.",
e);
}
}
}
/** {@inheritDoc} */
public void write(final DataOutput out) throws IOException {
out.writeByte(operator.ordinal());
out.writeInt(filters.size());
for (RowFilterInterface filter : filters) {
out.writeUTF(filter.getClass().getName());
filter.write(out);
}
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.io.ObjectWritable;
import org.apache.hadoop.io.Text;
/**
* Implementation of RowFilterInterface that represents a set of RowFilters
* which will be evaluated with a specified boolean operator MUST_PASS_ALL
* (!AND) or MUST_PASS_ONE (!OR). Since you can use RowFilterSets as children
* of RowFilterSet, you can create a hierarchy of filters to be evaluated.
*/
public class RowFilterSet implements RowFilterInterface {
/** set operator */
public static enum Operator {
/** !AND */
MUST_PASS_ALL,
/** !OR */
MUST_PASS_ONE
}
private Operator operator = Operator.MUST_PASS_ALL;
private Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
static final Log LOG = LogFactory.getLog(RowFilterSet.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RowFilterSet() {
super();
}
/**
* Constructor that takes a set of RowFilters. The default operator
* MUST_PASS_ALL is assumed.
*
* @param rowFilters
*/
public RowFilterSet(final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
}
/**
* Constructor that takes a set of RowFilters and an operator.
*
* @param operator Operator to process filter set with.
* @param rowFilters Set of row filters.
*/
public RowFilterSet(final Operator operator,
final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
this.operator = operator;
}
/** {@inheritDoc} */
public void validate(final Text[] columns) {
for (RowFilterInterface filter : filters) {
filter.validate(columns);
if (LOG.isDebugEnabled()) {
LOG.debug("Validated subfilter of type " +
filter.getClass().getSimpleName());
}
}
}
/** {@inheritDoc} */
public void reset() {
for (RowFilterInterface filter : filters) {
filter.reset();
if (LOG.isDebugEnabled()) {
LOG.debug("Reset subfilter of type " +
filter.getClass().getSimpleName());
}
}
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, Text rowKey) {
for (RowFilterInterface filter : filters) {
filter.rowProcessed(filtered, rowKey);
if (LOG.isDebugEnabled()) {
LOG.debug("Called rowProcessed on subfilter of type " +
filter.getClass().getSimpleName());
}
}
}
/** {@inheritDoc} */
public boolean processAlways() {
for (RowFilterInterface filter : filters) {
if (filter.processAlways()) {
if (LOG.isDebugEnabled()) {
LOG.debug("processAlways() is true due to subfilter of type " +
filter.getClass().getSimpleName());
}
return true;
}
}
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining()) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filterAllRemaining returning true due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
return true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining()) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filterAllRemaining returning false due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
return false;
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filterAllRemaining default returning " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filterRowKey(final Text rowKey) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterRowKey(rowKey)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filter(Text) will return true due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filter(Text) will return false due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterRowKey(rowKey);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filter(Text) returning " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filterColumn(final Text rowKey, final Text colKey,
final byte[] data) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() ||
filter.filterColumn(rowKey, colKey, data)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filter(Text, Text, byte[]) will" +
" return true due to subfilter of type " +
filter.getClass().getSimpleName());
}
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() &&
!filter.filterColumn(rowKey, colKey, data)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filter(Text, Text, byte[]) will" +
" return false due to subfilter of type " +
filter.getClass().getSimpleName());
}
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterColumn(rowKey, colKey, data);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filter(Text, Text, byte[]) returning " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filterRow(final SortedMap<Text, byte[]> columns) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterRow(columns)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPALL filterNotNull will return true due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filterRow(columns)) {
if (LOG.isDebugEnabled()) {
LOG.debug("op.MPONE filterNotNull will return false due" +
" to subfilter of type " + filter.getClass().getSimpleName());
}
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterRow(columns);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("filterNotNull returning " + result);
}
return result;
}
/** {@inheritDoc} */
public void readFields(final DataInput in) throws IOException {
Configuration conf = new HBaseConfiguration();
byte opByte = in.readByte();
operator = Operator.values()[opByte];
int size = in.readInt();
if (size > 0) {
filters = new HashSet<RowFilterInterface>();
for (int i = 0; i < size; i++) {
RowFilterInterface filter = (RowFilterInterface) ObjectWritable
.readObject(in, conf);
filters.add(filter);
if (LOG.isDebugEnabled()) {
LOG.debug("Successfully read in subfilter of type "
+ filter.getClass().getSimpleName());
}
}
}
}
/** {@inheritDoc} */
public void write(final DataOutput out) throws IOException {
Configuration conf = new HBaseConfiguration();
out.writeByte(operator.ordinal());
out.writeInt(filters.size());
for (RowFilterInterface filter : filters) {
ObjectWritable.writeObject(out, filter, RowFilterInterface.class, conf);
}
}
}

296
src/java/org/apache/hadoop/hbase/filter/StopRowFilter.java
View File

@ -1,148 +1,148 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
/**
* Implementation of RowFilterInterface that filters out rows greater than or
* equal to a specified rowKey.
*/
public class StopRowFilter implements RowFilterInterface {
protected Text stopRowKey;
static final Log LOG = LogFactory.getLog(StopRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public StopRowFilter() {
super();
}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public StopRowFilter(final Text stopRowKey) {
this.stopRowKey = stopRowKey;
}
/**
* An accessor for the stopRowKey
*
* @return the filter's stopRowKey
*/
public Text getStopRowKey() {
return this.stopRowKey;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final Text[] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, Text rowKey) {
// Doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
return false;
}
/** {@inheritDoc} */
public boolean filter(final Text rowKey) {
if (rowKey == null) {
if (this.stopRowKey == null) {
return true;
}
return false;
}
boolean result = this.stopRowKey.compareTo(rowKey) <= 0;
if (LOG.isDebugEnabled()) {
LOG.debug("Filter result for rowKey: " + rowKey + ". Result: " +
result);
}
return result;
}
/**
* {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling the rowKey-only version of filter.
*/
public boolean filter(@SuppressWarnings("unused") final Text rowKey,
@SuppressWarnings("unused") final Text colKey,
@SuppressWarnings("unused") final byte[] data) {
return filter(rowKey);
}
/** {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling filterAllRemaining().
*
* @param columns
*/
public boolean filterNotNull(@SuppressWarnings("unused")
final SortedMap<Text, byte[]> columns) {
return filterAllRemaining();
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
stopRowKey = new Text(in.readUTF());
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
out.writeUTF(stopRowKey.toString());
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
/**
* Implementation of RowFilterInterface that filters out rows greater than or
* equal to a specified rowKey.
*/
public class StopRowFilter implements RowFilterInterface {
protected Text stopRowKey;
static final Log LOG = LogFactory.getLog(StopRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public StopRowFilter() {
super();
}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public StopRowFilter(final Text stopRowKey) {
this.stopRowKey = stopRowKey;
}
/**
* An accessor for the stopRowKey
*
* @return the filter's stopRowKey
*/
public Text getStopRowKey() {
return this.stopRowKey;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final Text[] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, Text rowKey) {
// Doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
return false;
}
/** {@inheritDoc} */
public boolean filterRowKey(final Text rowKey) {
if (rowKey == null) {
if (this.stopRowKey == null) {
return true;
}
return false;
}
boolean result = this.stopRowKey.compareTo(rowKey) <= 0;
if (LOG.isDebugEnabled()) {
LOG.debug("Filter result for rowKey: " + rowKey + ". Result: " +
result);
}
return result;
}
/**
* {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling the rowKey-only version of filter.
*/
public boolean filterColumn(@SuppressWarnings("unused") final Text rowKey,
@SuppressWarnings("unused") final Text colKey,
@SuppressWarnings("unused") final byte[] data) {
return filterRowKey(rowKey);
}
/** {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling filterAllRemaining().
*
* @param columns
*/
public boolean filterRow(@SuppressWarnings("unused")
final SortedMap<Text, byte[]> columns) {
return filterAllRemaining();
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
stopRowKey = new Text(in.readUTF());
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
out.writeUTF(stopRowKey.toString());
}
}

362
src/java/org/apache/hadoop/hbase/filter/WhileMatchRowFilter.java
View File

@ -1,181 +1,181 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
/**
* WhileMatchRowFilter is a wrapper filter that filters everything after the
* first filtered row. Once the nested filter returns true for either of it's
* filter(..) methods or filterNotNull(SortedMap<Text, byte[]>), this wrapper's
* filterAllRemaining() will return true. All filtering methods will
* thereafter defer to the result of filterAllRemaining().
*/
public class WhileMatchRowFilter implements RowFilterInterface {
private boolean filterAllRemaining = false;
private RowFilterInterface filter;
static final Log LOG = LogFactory.getLog(WhileMatchRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public WhileMatchRowFilter() {
super();
}
/**
* Constructor
* @param filter
*/
public WhileMatchRowFilter(RowFilterInterface filter) {
this.filter = filter;
}
/**
* Returns the internal filter being wrapped
*
* @return the internal filter
*/
public RowFilterInterface getInternalFilter() {
return this.filter;
}
/** {@inheritDoc} */
public void reset() {
if (LOG.isDebugEnabled()) {
LOG.debug("Resetting.");
}
this.filterAllRemaining = false;
this.filter.reset();
}
/** {@inheritDoc} */
public boolean processAlways() {
return true;
}
/**
* Returns true once the nested filter has filtered out a row (returned true
* on a call to one of it's filtering methods). Until then it returns false.
*
* @return true/false whether the nested filter has returned true on a filter
* call.
*/
public boolean filterAllRemaining() {
return this.filterAllRemaining || this.filter.filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filter(final Text rowKey) {
changeFAR(this.filter.filter(rowKey));
boolean result = filterAllRemaining();
if (LOG.isDebugEnabled()) {
LOG.debug("Filter on rowKey:" + rowKey + ". Result = " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filter(final Text rowKey, final Text colKey,
final byte[] data) {
changeFAR(this.filter.filter(rowKey, colKey, data));
boolean result = filterAllRemaining();
if (LOG.isDebugEnabled()) {
LOG.debug("Filter on rowKey:" + rowKey + ", colKey: " + colKey +
", data: " + data + ". Result = " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filterNotNull(final SortedMap<Text, byte[]> columns) {
changeFAR(this.filter.filterNotNull(columns));
boolean result = filterAllRemaining();
if (LOG.isDebugEnabled()) {
LOG.debug("FilterNotNull on cols:" + columns + ". Result = " +
result);
}
return result;
}
/**
* Change filterAllRemaining from false to true if value is true, otherwise
* leave as is.
*
* @param value
*/
private void changeFAR(boolean value) {
this.filterAllRemaining = this.filterAllRemaining || value;
if (LOG.isDebugEnabled()) {
LOG.debug("this.filterAllRemaining is now: " +
this.filterAllRemaining);
}
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, Text rowKey) {
this.filter.rowProcessed(filtered, rowKey);
}
/** {@inheritDoc} */
public void validate(Text[] columns) {
this.filter.validate(columns);
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
String className = in.readUTF();
try {
this.filter = (RowFilterInterface)(Class.forName(className).
newInstance());
this.filter.readFields(in);
if (LOG.isDebugEnabled()) {
LOG.debug("Successfully read a sub-filter of type: " +
className);
}
} catch (InstantiationException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (IllegalAccessException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (ClassNotFoundException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
}
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
out.writeUTF(this.filter.getClass().getName());
this.filter.write(out);
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
/**
* WhileMatchRowFilter is a wrapper filter that filters everything after the
* first filtered row. Once the nested filter returns true for either of it's
* filter(..) methods or filterNotNull(SortedMap<Text, byte[]>), this wrapper's
* filterAllRemaining() will return true. All filtering methods will
* thereafter defer to the result of filterAllRemaining().
*/
public class WhileMatchRowFilter implements RowFilterInterface {
private boolean filterAllRemaining = false;
private RowFilterInterface filter;
static final Log LOG = LogFactory.getLog(WhileMatchRowFilter.class);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public WhileMatchRowFilter() {
super();
}
/**
* Constructor
* @param filter
*/
public WhileMatchRowFilter(RowFilterInterface filter) {
this.filter = filter;
}
/**
* Returns the internal filter being wrapped
*
* @return the internal filter
*/
public RowFilterInterface getInternalFilter() {
return this.filter;
}
/** {@inheritDoc} */
public void reset() {
if (LOG.isDebugEnabled()) {
LOG.debug("Resetting.");
}
this.filterAllRemaining = false;
this.filter.reset();
}
/** {@inheritDoc} */
public boolean processAlways() {
return true;
}
/**
* Returns true once the nested filter has filtered out a row (returned true
* on a call to one of it's filtering methods). Until then it returns false.
*
* @return true/false whether the nested filter has returned true on a filter
* call.
*/
public boolean filterAllRemaining() {
return this.filterAllRemaining || this.filter.filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filterRowKey(final Text rowKey) {
changeFAR(this.filter.filterRowKey(rowKey));
boolean result = filterAllRemaining();
if (LOG.isDebugEnabled()) {
LOG.debug("Filter on rowKey:" + rowKey + ". Result = " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filterColumn(final Text rowKey, final Text colKey,
final byte[] data) {
changeFAR(this.filter.filterColumn(rowKey, colKey, data));
boolean result = filterAllRemaining();
if (LOG.isDebugEnabled()) {
LOG.debug("Filter on rowKey:" + rowKey + ", colKey: " + colKey +
", data: " + data + ". Result = " + result);
}
return result;
}
/** {@inheritDoc} */
public boolean filterRow(final SortedMap<Text, byte[]> columns) {
changeFAR(this.filter.filterRow(columns));
boolean result = filterAllRemaining();
if (LOG.isDebugEnabled()) {
LOG.debug("FilterNotNull on cols:" + columns + ". Result = " +
result);
}
return result;
}
/**
* Change filterAllRemaining from false to true if value is true, otherwise
* leave as is.
*
* @param value
*/
private void changeFAR(boolean value) {
this.filterAllRemaining = this.filterAllRemaining || value;
if (LOG.isDebugEnabled()) {
LOG.debug("this.filterAllRemaining is now: " +
this.filterAllRemaining);
}
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, Text rowKey) {
this.filter.rowProcessed(filtered, rowKey);
}
/** {@inheritDoc} */
public void validate(Text[] columns) {
this.filter.validate(columns);
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
String className = in.readUTF();
try {
this.filter = (RowFilterInterface)(Class.forName(className).
newInstance());
this.filter.readFields(in);
if (LOG.isDebugEnabled()) {
LOG.debug("Successfully read a sub-filter of type: " +
className);
}
} catch (InstantiationException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (IllegalAccessException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (ClassNotFoundException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
}
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
out.writeUTF(this.filter.getClass().getName());
this.filter.write(out);
}
}

2
src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -1801,7 +1801,7 @@ public class HRegion implements HConstants {
moreToFollow = false;
}
filtered = filter == null ? false : filter.filterNotNull(results);
filtered = filter == null ? false : filter.filterRow(results);
if (moreToFollow) {
if (filter != null) {

568
src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java
View File

@ -1,284 +1,284 @@
/**
* Copyright 2008 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.TreeMap;
import java.util.SortedMap;
import java.util.Iterator;
import java.util.Map;
import java.util.List;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.HConstants;
/**
* Scanner scans both the memcache and the HStore
*/
class HStoreScanner implements InternalScanner {
static final Log LOG = LogFactory.getLog(HStoreScanner.class);
private InternalScanner[] scanners;
private TreeMap<Text, byte []>[] resultSets;
private HStoreKey[] keys;
private boolean wildcardMatch = false;
private boolean multipleMatchers = false;
private RowFilterInterface dataFilter;
private HStore store;
/** Create an Scanner with a handle on the memcache and HStore files. */
@SuppressWarnings("unchecked")
HStoreScanner(HStore store, Text[] targetCols, Text firstRow, long timestamp,
RowFilterInterface filter)
throws IOException {
this.store = store;
this.dataFilter = filter;
if (null != dataFilter) {
dataFilter.reset();
}
this.scanners = new InternalScanner[2];
this.resultSets = new TreeMap[scanners.length];
this.keys = new HStoreKey[scanners.length];
try {
scanners[0] = store.memcache.getScanner(timestamp, targetCols, firstRow);
scanners[1] = new StoreFileScanner(store, timestamp, targetCols, firstRow);
for (int i = 0; i < scanners.length; i++) {
if (scanners[i].isWildcardScanner()) {
this.wildcardMatch = true;
}
if (scanners[i].isMultipleMatchScanner()) {
this.multipleMatchers = true;
}
}
} catch(IOException e) {
for (int i = 0; i < this.scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
throw e;
}
// Advance to the first key in each scanner.
// All results will match the required column-set and scanTime.
for (int i = 0; i < scanners.length; i++) {
keys[i] = new HStoreKey();
resultSets[i] = new TreeMap<Text, byte []>();
if(scanners[i] != null && !scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
// As we have now successfully completed initialization, increment the
// activeScanner count.
store.activeScanners.incrementAndGet();
}
/** @return true if the scanner is a wild card scanner */
public boolean isWildcardScanner() {
return wildcardMatch;
}
/** @return true if the scanner is a multiple match scanner */
public boolean isMultipleMatchScanner() {
return multipleMatchers;
}
/** {@inheritDoc} */
public boolean next(HStoreKey key, SortedMap<Text, byte[]> results)
throws IOException {
// Filtered flag is set by filters. If a cell has been 'filtered out'
// -- i.e. it is not to be returned to the caller -- the flag is 'true'.
boolean filtered = true;
boolean moreToFollow = true;
while (filtered && moreToFollow) {
// Find the lowest-possible key.
Text chosenRow = null;
long chosenTimestamp = -1;
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
(chosenRow == null ||
(keys[i].getRow().compareTo(chosenRow) < 0) ||
((keys[i].getRow().compareTo(chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = new Text(keys[i].getRow());
chosenTimestamp = keys[i].getTimestamp();
}
}
// Filter whole row by row key?
filtered = dataFilter != null? dataFilter.filter(chosenRow) : false;
// Store the key and results for each sub-scanner. Merge them as
// appropriate.
if (chosenTimestamp >= 0 && !filtered) {
// Here we are setting the passed in key with current row+timestamp
key.setRow(chosenRow);
key.setVersion(chosenTimestamp);
key.setColumn(HConstants.EMPTY_TEXT);
// Keep list of deleted cell keys within this row. We need this
// because as we go through scanners, the delete record may be in an
// early scanner and then the same record with a non-delete, non-null
// value in a later. Without history of what we've seen, we'll return
// deleted values. This List should not ever grow too large since we
// are only keeping rows and columns that match those set on the
// scanner and which have delete values. If memory usage becomes a
// problem, could redo as bloom filter.
List<HStoreKey> deletes = new ArrayList<HStoreKey>();
for (int i = 0; i < scanners.length && !filtered; i++) {
while ((scanners[i] != null
&& !filtered
&& moreToFollow)
&& (keys[i].getRow().compareTo(chosenRow) == 0)) {
// If we are doing a wild card match or there are multiple
// matchers per column, we need to scan all the older versions of
// this row to pick up the rest of the family members
if (!wildcardMatch
&& !multipleMatchers
&& (keys[i].getTimestamp() != chosenTimestamp)) {
break;
}
// NOTE: We used to do results.putAll(resultSets[i]);
// but this had the effect of overwriting newer
// values with older ones. So now we only insert
// a result if the map does not contain the key.
HStoreKey hsk = new HStoreKey(key.getRow(), HConstants.EMPTY_TEXT,
key.getTimestamp());
for (Map.Entry<Text, byte[]> e : resultSets[i].entrySet()) {
hsk.setColumn(e.getKey());
if (HLogEdit.isDeleted(e.getValue())) {
if (!deletes.contains(hsk)) {
// Key changes as we cycle the for loop so add a copy to
// the set of deletes.
deletes.add(new HStoreKey(hsk));
}
} else if (!deletes.contains(hsk) &&
!filtered &&
moreToFollow &&
!results.containsKey(e.getKey())) {
if (dataFilter != null) {
// Filter whole row by column data?
filtered =
dataFilter.filter(chosenRow, e.getKey(), e.getValue());
if (filtered) {
results.clear();
break;
}
}
results.put(e.getKey(), e.getValue());
}
}
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
}
for (int i = 0; i < scanners.length; i++) {
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
(keys[i].getRow().compareTo(chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
moreToFollow = chosenTimestamp >= 0;
if (dataFilter != null) {
if (dataFilter.filterAllRemaining()) {
moreToFollow = false;
}
}
if (results.size() <= 0 && !filtered) {
// There were no results found for this row. Marked it as
// 'filtered'-out otherwise we will not move on to the next row.
filtered = true;
}
}
// If we got no results, then there is no more to follow.
if (results == null || results.size() <= 0) {
moreToFollow = false;
}
// Make sure scanners closed if no more results
if (!moreToFollow) {
for (int i = 0; i < scanners.length; i++) {
if (null != scanners[i]) {
closeScanner(i);
}
}
}
return moreToFollow;
}
/** Shut down a single scanner */
void closeScanner(int i) {
try {
try {
scanners[i].close();
} catch (IOException e) {
LOG.warn(store.storeName + " failed closing scanner " + i, e);
}
} finally {
scanners[i] = null;
keys[i] = null;
resultSets[i] = null;
}
}
/** {@inheritDoc} */
public void close() {
try {
for(int i = 0; i < scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
} finally {
store.updateActiveScanners();
}
}
/** {@inheritDoc} */
public Iterator<Map.Entry<HStoreKey, SortedMap<Text, byte[]>>> iterator() {
throw new UnsupportedOperationException("Unimplemented serverside. " +
"next(HStoreKey, StortedMap(...) is more efficient");
}
}
/**
* Copyright 2008 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.TreeMap;
import java.util.SortedMap;
import java.util.Iterator;
import java.util.Map;
import java.util.List;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.HConstants;
/**
* Scanner scans both the memcache and the HStore
*/
class HStoreScanner implements InternalScanner {
static final Log LOG = LogFactory.getLog(HStoreScanner.class);
private InternalScanner[] scanners;
private TreeMap<Text, byte []>[] resultSets;
private HStoreKey[] keys;
private boolean wildcardMatch = false;
private boolean multipleMatchers = false;
private RowFilterInterface dataFilter;
private HStore store;
/** Create an Scanner with a handle on the memcache and HStore files. */
@SuppressWarnings("unchecked")
HStoreScanner(HStore store, Text[] targetCols, Text firstRow, long timestamp,
RowFilterInterface filter)
throws IOException {
this.store = store;
this.dataFilter = filter;
if (null != dataFilter) {
dataFilter.reset();
}
this.scanners = new InternalScanner[2];
this.resultSets = new TreeMap[scanners.length];
this.keys = new HStoreKey[scanners.length];
try {
scanners[0] = store.memcache.getScanner(timestamp, targetCols, firstRow);
scanners[1] = new StoreFileScanner(store, timestamp, targetCols, firstRow);
for (int i = 0; i < scanners.length; i++) {
if (scanners[i].isWildcardScanner()) {
this.wildcardMatch = true;
}
if (scanners[i].isMultipleMatchScanner()) {
this.multipleMatchers = true;
}
}
} catch(IOException e) {
for (int i = 0; i < this.scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
throw e;
}
// Advance to the first key in each scanner.
// All results will match the required column-set and scanTime.
for (int i = 0; i < scanners.length; i++) {
keys[i] = new HStoreKey();
resultSets[i] = new TreeMap<Text, byte []>();
if(scanners[i] != null && !scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
// As we have now successfully completed initialization, increment the
// activeScanner count.
store.activeScanners.incrementAndGet();
}
/** @return true if the scanner is a wild card scanner */
public boolean isWildcardScanner() {
return wildcardMatch;
}
/** @return true if the scanner is a multiple match scanner */
public boolean isMultipleMatchScanner() {
return multipleMatchers;
}
/** {@inheritDoc} */
public boolean next(HStoreKey key, SortedMap<Text, byte[]> results)
throws IOException {
// Filtered flag is set by filters. If a cell has been 'filtered out'
// -- i.e. it is not to be returned to the caller -- the flag is 'true'.
boolean filtered = true;
boolean moreToFollow = true;
while (filtered && moreToFollow) {
// Find the lowest-possible key.
Text chosenRow = null;
long chosenTimestamp = -1;
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
(chosenRow == null ||
(keys[i].getRow().compareTo(chosenRow) < 0) ||
((keys[i].getRow().compareTo(chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = new Text(keys[i].getRow());
chosenTimestamp = keys[i].getTimestamp();
}
}
// Filter whole row by row key?
filtered = dataFilter != null? dataFilter.filterRowKey(chosenRow) : false;
// Store the key and results for each sub-scanner. Merge them as
// appropriate.
if (chosenTimestamp >= 0 && !filtered) {
// Here we are setting the passed in key with current row+timestamp
key.setRow(chosenRow);
key.setVersion(chosenTimestamp);
key.setColumn(HConstants.EMPTY_TEXT);
// Keep list of deleted cell keys within this row. We need this
// because as we go through scanners, the delete record may be in an
// early scanner and then the same record with a non-delete, non-null
// value in a later. Without history of what we've seen, we'll return
// deleted values. This List should not ever grow too large since we
// are only keeping rows and columns that match those set on the
// scanner and which have delete values. If memory usage becomes a
// problem, could redo as bloom filter.
List<HStoreKey> deletes = new ArrayList<HStoreKey>();
for (int i = 0; i < scanners.length && !filtered; i++) {
while ((scanners[i] != null
&& !filtered
&& moreToFollow)
&& (keys[i].getRow().compareTo(chosenRow) == 0)) {
// If we are doing a wild card match or there are multiple
// matchers per column, we need to scan all the older versions of
// this row to pick up the rest of the family members
if (!wildcardMatch
&& !multipleMatchers
&& (keys[i].getTimestamp() != chosenTimestamp)) {
break;
}
// NOTE: We used to do results.putAll(resultSets[i]);
// but this had the effect of overwriting newer
// values with older ones. So now we only insert
// a result if the map does not contain the key.
HStoreKey hsk = new HStoreKey(key.getRow(), HConstants.EMPTY_TEXT,
key.getTimestamp());
for (Map.Entry<Text, byte[]> e : resultSets[i].entrySet()) {
hsk.setColumn(e.getKey());
if (HLogEdit.isDeleted(e.getValue())) {
if (!deletes.contains(hsk)) {
// Key changes as we cycle the for loop so add a copy to
// the set of deletes.
deletes.add(new HStoreKey(hsk));
}
} else if (!deletes.contains(hsk) &&
!filtered &&
moreToFollow &&
!results.containsKey(e.getKey())) {
if (dataFilter != null) {
// Filter whole row by column data?
filtered =
dataFilter.filterColumn(chosenRow, e.getKey(), e.getValue());
if (filtered) {
results.clear();
break;
}
}
results.put(e.getKey(), e.getValue());
}
}
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
}
for (int i = 0; i < scanners.length; i++) {
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
(keys[i].getRow().compareTo(chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
moreToFollow = chosenTimestamp >= 0;
if (dataFilter != null) {
if (dataFilter.filterAllRemaining()) {
moreToFollow = false;
}
}
if (results.size() <= 0 && !filtered) {
// There were no results found for this row. Marked it as
// 'filtered'-out otherwise we will not move on to the next row.
filtered = true;
}
}
// If we got no results, then there is no more to follow.
if (results == null || results.size() <= 0) {
moreToFollow = false;
}
// Make sure scanners closed if no more results
if (!moreToFollow) {
for (int i = 0; i < scanners.length; i++) {
if (null != scanners[i]) {
closeScanner(i);
}
}
}
return moreToFollow;
}
/** Shut down a single scanner */
void closeScanner(int i) {
try {
try {
scanners[i].close();
} catch (IOException e) {
LOG.warn(store.storeName + " failed closing scanner " + i, e);
}
} finally {
scanners[i] = null;
keys[i] = null;
resultSets[i] = null;
}
}
/** {@inheritDoc} */
public void close() {
try {
for(int i = 0; i < scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
} finally {
store.updateActiveScanners();
}
}
/** {@inheritDoc} */
public Iterator<Map.Entry<HStoreKey, SortedMap<Text, byte[]>>> iterator() {
throw new UnsupportedOperationException("Unimplemented serverside. " +
"next(HStoreKey, StortedMap(...) is more efficient");
}
}

186
src/test/org/apache/hadoop/hbase/filter/TestInclusiveStopRowFilter.java
View File

@ -1,93 +1,93 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests the inclusive stop row filter
*/
public class TestInclusiveStopRowFilter extends TestCase {
private final Text STOP_ROW = new Text("stop_row");
private final Text GOOD_ROW = new Text("good_row");
private final Text PAST_STOP_ROW = new Text("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new InclusiveStopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new InclusiveStopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filter(GOOD_ROW));
assertFalse("Filtering on " + STOP_ROW, filter.filter(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filter(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filter(GOOD_ROW, null,
null));
assertFalse("Filtering on " + STOP_ROW, filter.filter(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filter(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterNotNull(null));
assertFalse("Filter a null", filter.filter(null));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests the inclusive stop row filter
*/
public class TestInclusiveStopRowFilter extends TestCase {
private final Text STOP_ROW = new Text("stop_row");
private final Text GOOD_ROW = new Text("good_row");
private final Text PAST_STOP_ROW = new Text("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new InclusiveStopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new InclusiveStopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filterRowKey(GOOD_ROW));
assertFalse("Filtering on " + STOP_ROW, filter.filterRowKey(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterRowKey(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filterColumn(GOOD_ROW, null,
null));
assertFalse("Filtering on " + STOP_ROW, filter.filterColumn(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterColumn(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterRow(null));
assertFalse("Filter a null", filter.filterRowKey(null));
}
}

196
src/test/org/apache/hadoop/hbase/filter/TestPageRowFilter.java
View File

@ -1,98 +1,98 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests for the page row filter
*/
public class TestPageRowFilter extends TestCase {
RowFilterInterface mainFilter;
final int ROW_LIMIT = 3;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new PageRowFilter(ROW_LIMIT);
}
/**
* test page size filter
* @throws Exception
*/
public void testPageSize() throws Exception {
pageSizeTests(mainFilter);
}
/**
* Test filter serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new PageRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
pageSizeTests(newFilter);
}
private void pageSizeTests(RowFilterInterface filter) throws Exception {
testFiltersBeyondPageSize(filter, ROW_LIMIT);
// Test reset works by going in again.
filter.reset();
testFiltersBeyondPageSize(filter, ROW_LIMIT);
}
private void testFiltersBeyondPageSize(final RowFilterInterface filter,
final int pageSize) {
for (int i = 0; i < (pageSize * 2); i++) {
Text row = new Text(Integer.toString(i));
boolean filterOut = filter.filter(row);
if (!filterOut) {
assertFalse("Disagrees with 'filter'", filter.filterAllRemaining());
} else {
// Once we have all for a page, calls to filterAllRemaining should
// stay true.
assertTrue("Disagrees with 'filter'", filter.filterAllRemaining());
assertTrue(i >= pageSize);
}
filter.rowProcessed(filterOut, row);
}
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests for the page row filter
*/
public class TestPageRowFilter extends TestCase {
RowFilterInterface mainFilter;
final int ROW_LIMIT = 3;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new PageRowFilter(ROW_LIMIT);
}
/**
* test page size filter
* @throws Exception
*/
public void testPageSize() throws Exception {
pageSizeTests(mainFilter);
}
/**
* Test filter serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new PageRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
pageSizeTests(newFilter);
}
private void pageSizeTests(RowFilterInterface filter) throws Exception {
testFiltersBeyondPageSize(filter, ROW_LIMIT);
// Test reset works by going in again.
filter.reset();
testFiltersBeyondPageSize(filter, ROW_LIMIT);
}
private void testFiltersBeyondPageSize(final RowFilterInterface filter,
final int pageSize) {
for (int i = 0; i < (pageSize * 2); i++) {
Text row = new Text(Integer.toString(i));
boolean filterOut = filter.filterRowKey(row);
if (!filterOut) {
assertFalse("Disagrees with 'filter'", filter.filterAllRemaining());
} else {
// Once we have all for a page, calls to filterAllRemaining should
// stay true.
assertTrue("Disagrees with 'filter'", filter.filterAllRemaining());
assertTrue(i >= pageSize);
}
filter.rowProcessed(filterOut, row);
}
}
}

394
src/test/org/apache/hadoop/hbase/filter/TestRegExpRowFilter.java
View File

@ -1,197 +1,197 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.TreeMap;
import junit.framework.TestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
import org.apache.hadoop.io.Text;
/**
* Tests for regular expression row filter
*/
public class TestRegExpRowFilter extends TestCase {
TreeMap<Text, byte []> colvalues;
RowFilterInterface mainFilter;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
final String HOST_PREFIX = "org.apache.site-";
static byte [] GOOD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
this.colvalues = new TreeMap<Text, byte[]>();
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(new Text(new String(new char [] {c})), GOOD_BYTES);
}
this.mainFilter = new RegExpRowFilter(HOST_PREFIX + ".*", colvalues);
}
/**
* Tests filtering using a regex on the row key
* @throws Exception
*/
public void testRegexOnRow() throws Exception {
regexRowTests(mainFilter);
}
/**
* Tests filtering using a regex on row and colum
* @throws Exception
*/
public void testRegexOnRowAndColumn() throws Exception {
regexRowColumnTests(mainFilter);
}
/**
* Only return values that are not null
* @throws Exception
*/
public void testFilterNotNull() throws Exception {
filterNotNullTests(mainFilter);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RegExpRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running all test.
regexRowTests(newFilter);
newFilter.reset();
regexRowColumnTests(newFilter);
newFilter.reset();
filterNotNullTests(newFilter);
}
private void regexRowTests(RowFilterInterface filter) throws Exception {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
Text t = createRow(c);
assertFalse("Failed with characer " + c, filter.filter(t));
}
String yahooSite = "com.yahoo.www";
assertTrue("Failed with character " +
yahooSite, filter.filter(new Text(yahooSite)));
}
private void regexRowColumnTests(RowFilterInterface filter)
throws UnsupportedEncodingException {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
Text t = createRow(c);
for (Map.Entry<Text, byte []> e: this.colvalues.entrySet()) {
assertFalse("Failed on " + c,
filter.filter(t, e.getKey(), e.getValue()));
}
}
// Try a row and column I know will pass.
char c = 'c';
Text r = createRow(c);
Text col = new Text(Character.toString(c));
assertFalse("Failed with character " + c,
filter.filter(r, col, GOOD_BYTES));
// Do same but with bad bytes.
assertTrue("Failed with character " + c,
filter.filter(r, col, "badbytes".getBytes(HConstants.UTF8_ENCODING)));
// Do with good bytes but bad column name. Should not filter out.
assertFalse("Failed with character " + c,
filter.filter(r, new Text("badcolumn"), GOOD_BYTES));
// Good column, good bytes but bad row.
assertTrue("Failed with character " + c,
filter.filter(new Text("bad row"), new Text("badcolumn"), GOOD_BYTES));
}
private void filterNotNullTests(RowFilterInterface filter) throws Exception {
// Modify the filter to expect certain columns to be null:
// Expecting a row WITH columnKeys: a-d, WITHOUT columnKey: e
((RegExpRowFilter)filter).setColumnFilter(new Text(new String(new char[] {
LAST_CHAR })), null);
char secondToLast = (char)(LAST_CHAR - 1);
char thirdToLast = (char)(LAST_CHAR - 2);
// Modify the row to be missing an expected columnKey (d)
colvalues.remove(new Text(new String(new char[] { secondToLast })));
// Try a row that is missing an expected columnKey.
// Testing row with columnKeys: a-c
assertTrue("Failed with last columnKey " + thirdToLast, filter.
filterNotNull(colvalues));
// Try a row that has all expected columnKeys, and NO null-expected
// columnKeys.
// Testing row with columnKeys: a-d
colvalues.put(new Text(new String(new char[] { secondToLast })),
GOOD_BYTES);
assertFalse("Failed with last columnKey " + secondToLast, filter.
filterNotNull(colvalues));
// Try a row that has all expected columnKeys AND a null-expected columnKey.
// Testing row with columnKeys: a-e
colvalues.put(new Text(new String(new char[] { LAST_CHAR })), GOOD_BYTES);
assertTrue("Failed with last columnKey " + LAST_CHAR, filter.
filterNotNull(colvalues));
// Try a row that has all expected columnKeys and a null-expected columnKey
// that maps to a null value.
// Testing row with columnKeys: a-e, e maps to null
colvalues.put(new Text(new String(new char[] { LAST_CHAR })),
HLogEdit.deleteBytes.get());
assertFalse("Failed with last columnKey " + LAST_CHAR + " mapping to null.",
filter.filterNotNull(colvalues));
}
private Text createRow(final char c) {
return new Text(HOST_PREFIX + Character.toString(c));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.TreeMap;
import junit.framework.TestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
import org.apache.hadoop.io.Text;
/**
* Tests for regular expression row filter
*/
public class TestRegExpRowFilter extends TestCase {
TreeMap<Text, byte []> colvalues;
RowFilterInterface mainFilter;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
final String HOST_PREFIX = "org.apache.site-";
static byte [] GOOD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
this.colvalues = new TreeMap<Text, byte[]>();
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(new Text(new String(new char [] {c})), GOOD_BYTES);
}
this.mainFilter = new RegExpRowFilter(HOST_PREFIX + ".*", colvalues);
}
/**
* Tests filtering using a regex on the row key
* @throws Exception
*/
public void testRegexOnRow() throws Exception {
regexRowTests(mainFilter);
}
/**
* Tests filtering using a regex on row and colum
* @throws Exception
*/
public void testRegexOnRowAndColumn() throws Exception {
regexRowColumnTests(mainFilter);
}
/**
* Only return values that are not null
* @throws Exception
*/
public void testFilterNotNull() throws Exception {
filterNotNullTests(mainFilter);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RegExpRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running all test.
regexRowTests(newFilter);
newFilter.reset();
regexRowColumnTests(newFilter);
newFilter.reset();
filterNotNullTests(newFilter);
}
private void regexRowTests(RowFilterInterface filter) throws Exception {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
Text t = createRow(c);
assertFalse("Failed with characer " + c, filter.filterRowKey(t));
}
String yahooSite = "com.yahoo.www";
assertTrue("Failed with character " +
yahooSite, filter.filterRowKey(new Text(yahooSite)));
}
private void regexRowColumnTests(RowFilterInterface filter)
throws UnsupportedEncodingException {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
Text t = createRow(c);
for (Map.Entry<Text, byte []> e: this.colvalues.entrySet()) {
assertFalse("Failed on " + c,
filter.filterColumn(t, e.getKey(), e.getValue()));
}
}
// Try a row and column I know will pass.
char c = 'c';
Text r = createRow(c);
Text col = new Text(Character.toString(c));
assertFalse("Failed with character " + c,
filter.filterColumn(r, col, GOOD_BYTES));
// Do same but with bad bytes.
assertTrue("Failed with character " + c,
filter.filterColumn(r, col, "badbytes".getBytes(HConstants.UTF8_ENCODING)));
// Do with good bytes but bad column name. Should not filter out.
assertFalse("Failed with character " + c,
filter.filterColumn(r, new Text("badcolumn"), GOOD_BYTES));
// Good column, good bytes but bad row.
assertTrue("Failed with character " + c,
filter.filterColumn(new Text("bad row"), new Text("badcolumn"), GOOD_BYTES));
}
private void filterNotNullTests(RowFilterInterface filter) throws Exception {
// Modify the filter to expect certain columns to be null:
// Expecting a row WITH columnKeys: a-d, WITHOUT columnKey: e
((RegExpRowFilter)filter).setColumnFilter(new Text(new String(new char[] {
LAST_CHAR })), null);
char secondToLast = (char)(LAST_CHAR - 1);
char thirdToLast = (char)(LAST_CHAR - 2);
// Modify the row to be missing an expected columnKey (d)
colvalues.remove(new Text(new String(new char[] { secondToLast })));
// Try a row that is missing an expected columnKey.
// Testing row with columnKeys: a-c
assertTrue("Failed with last columnKey " + thirdToLast, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys, and NO null-expected
// columnKeys.
// Testing row with columnKeys: a-d
colvalues.put(new Text(new String(new char[] { secondToLast })),
GOOD_BYTES);
assertFalse("Failed with last columnKey " + secondToLast, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys AND a null-expected columnKey.
// Testing row with columnKeys: a-e
colvalues.put(new Text(new String(new char[] { LAST_CHAR })), GOOD_BYTES);
assertTrue("Failed with last columnKey " + LAST_CHAR, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys and a null-expected columnKey
// that maps to a null value.
// Testing row with columnKeys: a-e, e maps to null
colvalues.put(new Text(new String(new char[] { LAST_CHAR })),
HLogEdit.deleteBytes.get());
assertFalse("Failed with last columnKey " + LAST_CHAR + " mapping to null.",
filter.filterRow(colvalues));
}
private Text createRow(final char c) {
return new Text(HOST_PREFIX + Character.toString(c));
}
}

374
src/test/org/apache/hadoop/hbase/filter/TestRowFilterSet.java
View File

@ -1,187 +1,187 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;
import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests filter sets
*/
public class TestRowFilterSet extends TestCase {
RowFilterInterface filterMPALL;
RowFilterInterface filterMPONE;
static final int MAX_PAGES = 5;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
TreeMap<Text, byte[]> colvalues;
static byte[] GOOD_BYTES = null;
static byte[] BAD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
BAD_BYTES = "def".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
colvalues = new TreeMap<Text, byte[]>();
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(new Text(new String(new char [] {c})), GOOD_BYTES);
}
Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
filters.add(new PageRowFilter(MAX_PAGES));
filters.add(new RegExpRowFilter(".*regex.*", colvalues));
filters.add(new WhileMatchRowFilter(new StopRowFilter(new Text("yyy"))));
filters.add(new WhileMatchRowFilter(new RegExpRowFilter(".*match.*")));
filterMPALL = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ALL,
filters);
filterMPONE = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ONE,
filters);
}
/**
* Test "must pass one"
* @throws Exception
*/
public void testMPONE() throws Exception {
MPONETests(filterMPONE);
}
/**
* Test "must pass all"
* @throws Exception
*/
public void testMPALL() throws Exception {
MPALLTests(filterMPALL);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose filterMPALL to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
filterMPALL.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filterMPALL.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RowFilterSet();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
MPALLTests(newFilter);
}
private void MPONETests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause the WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", false);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", false);
// Accept several more rows such that PageRowFilter will exceed its limit.
for (int i=0; i<=MAX_PAGES-3; i++)
filter.rowProcessed(false, new Text("unimportant_key"));
// A row that should cause the RegExpRowFilter to filter this row, making
// all the filters return true and thus the RowFilterSet as well.
RFSAssertion(filter, "bad_column", true);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
}
private void MPALLTests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail.
boolean filtered = filter.filter(new Text("regex_match"),
new Text(new String(new char[] { FIRST_CHAR })), BAD_BYTES);
assertTrue("Filtering on 'regex_match' and bad column data.", filtered);
filterMPALL.rowProcessed(filtered, new Text("regex_match"));
}
private void RFSAssertion(RowFilterInterface filter, String toTest,
boolean assertTrue) throws Exception {
Text testText = new Text(toTest);
boolean filtered = filter.filter(testText);
assertTrue("Filtering on '" + toTest + "'",
assertTrue? filtered : !filtered);
filter.rowProcessed(filtered, testText);
}
private void RFSAssertReset(RowFilterInterface filter) throws Exception{
assertTrue(filter.filterAllRemaining());
// Reset for continued testing
filter.reset();
assertFalse(filter.filterAllRemaining());
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;
import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests filter sets
*/
public class TestRowFilterSet extends TestCase {
RowFilterInterface filterMPALL;
RowFilterInterface filterMPONE;
static final int MAX_PAGES = 5;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
TreeMap<Text, byte[]> colvalues;
static byte[] GOOD_BYTES = null;
static byte[] BAD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
BAD_BYTES = "def".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
colvalues = new TreeMap<Text, byte[]>();
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(new Text(new String(new char [] {c})), GOOD_BYTES);
}
Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
filters.add(new PageRowFilter(MAX_PAGES));
filters.add(new RegExpRowFilter(".*regex.*", colvalues));
filters.add(new WhileMatchRowFilter(new StopRowFilter(new Text("yyy"))));
filters.add(new WhileMatchRowFilter(new RegExpRowFilter(".*match.*")));
filterMPALL = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ALL,
filters);
filterMPONE = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ONE,
filters);
}
/**
* Test "must pass one"
* @throws Exception
*/
public void testMPONE() throws Exception {
MPONETests(filterMPONE);
}
/**
* Test "must pass all"
* @throws Exception
*/
public void testMPALL() throws Exception {
MPALLTests(filterMPALL);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose filterMPALL to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
filterMPALL.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filterMPALL.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RowFilterSet();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
MPALLTests(newFilter);
}
private void MPONETests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause the WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", false);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", false);
// Accept several more rows such that PageRowFilter will exceed its limit.
for (int i=0; i<=MAX_PAGES-3; i++)
filter.rowProcessed(false, new Text("unimportant_key"));
// A row that should cause the RegExpRowFilter to filter this row, making
// all the filters return true and thus the RowFilterSet as well.
RFSAssertion(filter, "bad_column", true);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
}
private void MPALLTests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail.
boolean filtered = filter.filterColumn(new Text("regex_match"),
new Text(new String(new char[] { FIRST_CHAR })), BAD_BYTES);
assertTrue("Filtering on 'regex_match' and bad column data.", filtered);
filterMPALL.rowProcessed(filtered, new Text("regex_match"));
}
private void RFSAssertion(RowFilterInterface filter, String toTest,
boolean assertTrue) throws Exception {
Text testText = new Text(toTest);
boolean filtered = filter.filterRowKey(testText);
assertTrue("Filtering on '" + toTest + "'",
assertTrue? filtered : !filtered);
filter.rowProcessed(filtered, testText);
}
private void RFSAssertReset(RowFilterInterface filter) throws Exception{
assertTrue(filter.filterAllRemaining());
// Reset for continued testing
filter.reset();
assertFalse(filter.filterAllRemaining());
}
}

186
src/test/org/apache/hadoop/hbase/filter/TestStopRowFilter.java
View File

@ -1,93 +1,93 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests the stop row filter
*/
public class TestStopRowFilter extends TestCase {
private final Text STOP_ROW = new Text("stop_row");
private final Text GOOD_ROW = new Text("good_row");
private final Text PAST_STOP_ROW = new Text("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new StopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new StopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filter(GOOD_ROW));
assertTrue("Filtering on " + STOP_ROW, filter.filter(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filter(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filter(GOOD_ROW, null,
null));
assertTrue("Filtering on " + STOP_ROW, filter.filter(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filter(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterNotNull(null));
assertFalse("Filter a null", filter.filter(null));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
/**
* Tests the stop row filter
*/
public class TestStopRowFilter extends TestCase {
private final Text STOP_ROW = new Text("stop_row");
private final Text GOOD_ROW = new Text("good_row");
private final Text PAST_STOP_ROW = new Text("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new StopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new StopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filterRowKey(GOOD_ROW));
assertTrue("Filtering on " + STOP_ROW, filter.filterRowKey(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterRowKey(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filterColumn(GOOD_ROW, null,
null));
assertTrue("Filtering on " + STOP_ROW, filter.filterColumn(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterColumn(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterRow(null));
assertFalse("Filter a null", filter.filterRowKey(null));
}
}

298
src/test/org/apache/hadoop/hbase/filter/TestWhileMatchRowFilter.java
View File

@ -1,149 +1,149 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import junit.framework.TestCase;
import org.apache.hadoop.io.Text;
/**
* Tests for the while-match filter
*/
public class TestWhileMatchRowFilter extends TestCase {
WhileMatchRowFilter wmStopRowFilter;
WhileMatchRowFilter wmRegExpRowFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
wmStopRowFilter = new WhileMatchRowFilter(new StopRowFilter(
new Text("s")));
wmRegExpRowFilter = new WhileMatchRowFilter(new RegExpRowFilter(
".*regex.*"));
}
/**
* Tests while match stop row
* @throws Exception
*/
public void testWhileMatchStopRow() throws Exception {
whileMatchStopRowTests(wmStopRowFilter);
}
/**
* Tests while match regex
* @throws Exception
*/
public void testWhileMatchRegExp() throws Exception {
whileMatchRegExpTests(wmRegExpRowFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose wmRegExpRowFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
wmRegExpRowFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose wmRegExpRowFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
WhileMatchRowFilter newFilter = new WhileMatchRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
whileMatchRegExpTests(newFilter);
}
private void whileMatchStopRowTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "apples";
assertFalse("filter: '" + toTest + "'", filter.filter(new Text(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filter(new Text(
toTest)));
// Test cases that should fail the row
toTest = "tuna";
assertTrue("filter: '" + toTest + "'", filter.filter(new Text(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filter(new Text(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filterNotNull for functionality only (no switch-cases)
assertFalse("filter: filterNotNull", filter.filterNotNull(null));
}
private void whileMatchRegExpTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "regex_match";
assertFalse("filter: '" + toTest + "'", filter.filter(new Text(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filter(new Text(
toTest)));
// Test cases that should fail the row
toTest = "not_a_match";
assertTrue("filter: '" + toTest + "'", filter.filter(new Text(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filter(new Text(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filter(Text, Text, byte[]) for functionality only (no switch-cases)
toTest = "asdf_regex_hjkl";
assertFalse("filter: '" + toTest + "'", filter.filter(new Text(toTest),
null, null));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import junit.framework.TestCase;
import org.apache.hadoop.io.Text;
/**
* Tests for the while-match filter
*/
public class TestWhileMatchRowFilter extends TestCase {
WhileMatchRowFilter wmStopRowFilter;
WhileMatchRowFilter wmRegExpRowFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
wmStopRowFilter = new WhileMatchRowFilter(new StopRowFilter(
new Text("s")));
wmRegExpRowFilter = new WhileMatchRowFilter(new RegExpRowFilter(
".*regex.*"));
}
/**
* Tests while match stop row
* @throws Exception
*/
public void testWhileMatchStopRow() throws Exception {
whileMatchStopRowTests(wmStopRowFilter);
}
/**
* Tests while match regex
* @throws Exception
*/
public void testWhileMatchRegExp() throws Exception {
whileMatchRegExpTests(wmRegExpRowFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose wmRegExpRowFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
wmRegExpRowFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose wmRegExpRowFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
WhileMatchRowFilter newFilter = new WhileMatchRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
whileMatchRegExpTests(newFilter);
}
private void whileMatchStopRowTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "apples";
assertFalse("filter: '" + toTest + "'", filter.filterRowKey(new Text(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(new Text(
toTest)));
// Test cases that should fail the row
toTest = "tuna";
assertTrue("filter: '" + toTest + "'", filter.filterRowKey(new Text(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(new Text(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filterNotNull for functionality only (no switch-cases)
assertFalse("filter: filterNotNull", filter.filterRow(null));
}
private void whileMatchRegExpTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "regex_match";
assertFalse("filter: '" + toTest + "'", filter.filterRowKey(new Text(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(new Text(
toTest)));
// Test cases that should fail the row
toTest = "not_a_match";
assertTrue("filter: '" + toTest + "'", filter.filterRowKey(new Text(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(new Text(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filter(Text, Text, byte[]) for functionality only (no switch-cases)
toTest = "asdf_regex_hjkl";
assertFalse("filter: '" + toTest + "'", filter.filterColumn(new Text(toTest),
null, null));
}
}