Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-638 Purge \r from src 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@659249 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2008-05-22 20:49:25 +00:00
parent 192e438289
commit f63ecc9a06
17 changed files with 3332 additions and 3331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -19,6 +19,7 @@ Hbase Change Log
HBASE-630 Default hbase.rootdir is garbage
HBASE-589 Remove references to deprecated methods in Hadoop once
hadoop-0.17.0 is released
HBASE-638 Purge \r from src
IMPROVEMENTS
HBASE-559 MR example job to count table rows

1958
src/java/org/apache/hadoop/hbase/client/HTable.java
View File

File diff suppressed because it is too large Load Diff

110
src/java/org/apache/hadoop/hbase/filter/InclusiveStopRowFilter.java
View File

@ -1,55 +1,55 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Subclass of StopRowFilter that filters rows > the stop row,
* making it include up to the last row but no further.
*/
public class InclusiveStopRowFilter extends StopRowFilter{
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public InclusiveStopRowFilter() {super();}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public InclusiveStopRowFilter(final byte [] stopRowKey) {
super(stopRowKey);
}
/** {@inheritDoc} */
@Override
public boolean filterRowKey(final byte [] rowKey) {
if (rowKey == null) {
if (getStopRowKey() == null) {
return true;
}
return false;
}
return Bytes.compareTo(getStopRowKey(), rowKey) < 0;
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Subclass of StopRowFilter that filters rows > the stop row,
* making it include up to the last row but no further.
*/
public class InclusiveStopRowFilter extends StopRowFilter{
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public InclusiveStopRowFilter() {super();}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public InclusiveStopRowFilter(final byte [] stopRowKey) {
super(stopRowKey);
}
/** {@inheritDoc} */
@Override
public boolean filterRowKey(final byte [] rowKey) {
if (rowKey == null) {
if (getStopRowKey() == null) {
return true;
}
return false;
}
return Bytes.compareTo(getStopRowKey(), rowKey) < 0;
}
}

288
src/java/org/apache/hadoop/hbase/filter/PageRowFilter.java
View File

@ -1,145 +1,145 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
/**
* Implementation of RowFilterInterface that limits results to a specific page
* size. It terminates scanning once the number of filter-passed results is >=
* the given page size.
*
* <p>
* Note that this filter cannot guarantee that the number of results returned
* to a client are <= page size. This is because the filter is applied
* separately on different region servers. It does however optimize the scan of
* individual HRegions by making sure that the page size is never exceeded
* locally.
* </p>
*/
public class PageRowFilter implements RowFilterInterface {
private long pageSize = Long.MAX_VALUE;
private int rowsAccepted = 0;
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public PageRowFilter() {
super();
}
/**
* Constructor that takes a maximum page size.
*
* @param pageSize Maximum result size.
*/
public PageRowFilter(final long pageSize) {
this.pageSize = pageSize;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final byte [][] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
rowsAccepted = 0;
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered,
@SuppressWarnings("unused") byte [] rowKey) {
if (!filtered) {
this.rowsAccepted++;
}
}
/**
*
* {@inheritDoc}
*/
public boolean processAlways() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
return this.rowsAccepted > this.pageSize;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRowKey(@SuppressWarnings("unused") final byte [] r) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filterColumn(@SuppressWarnings("unused") final byte [] rowKey,
@SuppressWarnings("unused") final byte [] colKey,
@SuppressWarnings("unused") final byte[] data) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filterRow(@SuppressWarnings("unused")
final SortedMap<byte [], byte[]> columns) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
this.pageSize = in.readLong();
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
out.writeLong(pageSize);
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
/**
* Implementation of RowFilterInterface that limits results to a specific page
* size. It terminates scanning once the number of filter-passed results is >=
* the given page size.
*
* <p>
* Note that this filter cannot guarantee that the number of results returned
* to a client are <= page size. This is because the filter is applied
* separately on different region servers. It does however optimize the scan of
* individual HRegions by making sure that the page size is never exceeded
* locally.
* </p>
*/
public class PageRowFilter implements RowFilterInterface {
private long pageSize = Long.MAX_VALUE;
private int rowsAccepted = 0;
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public PageRowFilter() {
super();
}
/**
* Constructor that takes a maximum page size.
*
* @param pageSize Maximum result size.
*/
public PageRowFilter(final long pageSize) {
this.pageSize = pageSize;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final byte [][] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
rowsAccepted = 0;
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered,
@SuppressWarnings("unused") byte [] rowKey) {
if (!filtered) {
this.rowsAccepted++;
}
}
/**
*
* {@inheritDoc}
*/
public boolean processAlways() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
return this.rowsAccepted > this.pageSize;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRowKey(@SuppressWarnings("unused") final byte [] r) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filterColumn(@SuppressWarnings("unused") final byte [] rowKey,
@SuppressWarnings("unused") final byte [] colKey,
@SuppressWarnings("unused") final byte[] data) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public boolean filterRow(@SuppressWarnings("unused")
final SortedMap<byte [], byte[]> columns) {
return filterAllRemaining();
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
this.pageSize = in.readLong();
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
out.writeLong(pageSize);
}
}

634
src/java/org/apache/hadoop/hbase/filter/RegExpRowFilter.java
View File

@ -1,318 +1,318 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Implementation of RowFilterInterface that can filter by rowkey regular
* expression and/or individual column values (equals comparison only). Multiple
* column filters imply an implicit conjunction of filter criteria.
*
* Note that column value filtering in this interface has been replaced by
* {@link ColumnValueFilter}.
*/
public class RegExpRowFilter implements RowFilterInterface {
private Pattern rowKeyPattern = null;
private String rowKeyRegExp = null;
@Deprecated
private Map<byte [], byte[]> equalsMap =
new TreeMap<byte [], byte[]>(Bytes.BYTES_COMPARATOR);
@Deprecated
private Set<byte []> nullColumns =
new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RegExpRowFilter() {
super();
}
/**
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
*/
public RegExpRowFilter(final String rowKeyRegExp) {
this.rowKeyRegExp = rowKeyRegExp;
}
/**
* @deprecated Column filtering has been replaced by {@link ColumnValueFilter}
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
* @param columnFilter
*/
@Deprecated
public RegExpRowFilter(final String rowKeyRegExp,
final Map<byte [], byte[]> columnFilter) {
this.rowKeyRegExp = rowKeyRegExp;
this.setColumnFilters(columnFilter);
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, byte [] rowKey) {
//doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/**
* @deprecated Column filtering has been replaced by {@link ColumnValueFilter}
* Specify a value that must be matched for the given column.
*
* @param colKey
* the column to match on
* @param value
* the value that must equal the stored value.
*/
@Deprecated
public void setColumnFilter(final byte [] colKey, final byte[] value) {
if (value == null) {
nullColumns.add(colKey);
} else {
equalsMap.put(colKey, value);
}
}
/**
* @deprecated Column filtering has been replaced by {@link ColumnValueFilter}
* Set column filters for a number of columns.
*
* @param columnFilter
* Map of columns with value criteria.
*/
@Deprecated
public void setColumnFilters(final Map<byte [], byte[]> columnFilter) {
if (null == columnFilter) {
nullColumns.clear();
equalsMap.clear();
} else {
for (Entry<byte [], byte[]> entry : columnFilter.entrySet()) {
setColumnFilter(entry.getKey(), entry.getValue());
}
}
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRowKey(final byte [] rowKey) {
return (filtersByRowKey() && rowKey != null)?
!getRowKeyPattern().matcher(Bytes.toString(rowKey)).matches():
false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterColumn(final byte [] rowKey, final byte [] colKey,
final byte[] data) {
if (filterRowKey(rowKey)) {
return true;
}
if (filtersByColumnValue()) {
byte[] filterValue = equalsMap.get(colKey);
if (null != filterValue) {
return !Arrays.equals(filterValue, data);
}
}
if (nullColumns.contains(colKey)) {
if (data != null && !HLogEdit.isDeleted(data)) {
return true;
}
}
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRow(final SortedMap<byte [], byte[]> columns) {
for (Entry<byte [], byte[]> col : columns.entrySet()) {
if (nullColumns.contains(col.getKey())
&& !HLogEdit.isDeleted(col.getValue())) {
return true;
}
}
for (byte [] col : equalsMap.keySet()) {
if (!columns.containsKey(col)) {
return true;
}
}
return false;
}
@Deprecated
private boolean filtersByColumnValue() {
return equalsMap != null && equalsMap.size() > 0;
}
private boolean filtersByRowKey() {
return null != rowKeyPattern || null != rowKeyRegExp;
}
private String getRowKeyRegExp() {
if (null == rowKeyRegExp && rowKeyPattern != null) {
rowKeyRegExp = rowKeyPattern.toString();
}
return rowKeyRegExp;
}
private Pattern getRowKeyPattern() {
if (rowKeyPattern == null && rowKeyRegExp != null) {
rowKeyPattern = Pattern.compile(rowKeyRegExp);
}
return rowKeyPattern;
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
boolean hasRowKeyPattern = in.readBoolean();
if (hasRowKeyPattern) {
rowKeyRegExp = in.readUTF();
}
// equals map
equalsMap.clear();
int size = in.readInt();
for (int i = 0; i < size; i++) {
byte [] key = Bytes.readByteArray(in);
int len = in.readInt();
byte[] value = null;
if (len >= 0) {
value = new byte[len];
in.readFully(value);
}
setColumnFilter(key, value);
}
// nullColumns
nullColumns.clear();
size = in.readInt();
for (int i = 0; i < size; i++) {
setColumnFilter(Bytes.readByteArray(in), null);
}
}
/**
*
* {@inheritDoc}
*/
public void validate(final byte [][] columns) {
Set<byte []> invalids = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
for (byte [] colKey : getFilterColumns()) {
boolean found = false;
for (byte [] col : columns) {
if (Bytes.equals(col, colKey)) {
found = true;
break;
}
}
if (!found) {
invalids.add(colKey);
}
}
if (invalids.size() > 0) {
throw new InvalidRowFilterException(String.format(
"RowFilter contains criteria on columns %s not in %s", invalids,
Arrays.toString(columns)));
}
}
@Deprecated
private Set<byte []> getFilterColumns() {
Set<byte []> cols = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
cols.addAll(equalsMap.keySet());
cols.addAll(nullColumns);
return cols;
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
if (!filtersByRowKey()) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeUTF(getRowKeyRegExp());
}
// equalsMap
out.writeInt(equalsMap.size());
for (Entry<byte [], byte[]> entry : equalsMap.entrySet()) {
Bytes.writeByteArray(out, entry.getKey());
byte[] value = entry.getValue();
out.writeInt(value.length);
out.write(value);
}
// null columns
out.writeInt(nullColumns.size());
for (byte [] col : nullColumns) {
Bytes.writeByteArray(out, col);
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Implementation of RowFilterInterface that can filter by rowkey regular
* expression and/or individual column values (equals comparison only). Multiple
* column filters imply an implicit conjunction of filter criteria.
*
* Note that column value filtering in this interface has been replaced by
* {@link ColumnValueFilter}.
*/
public class RegExpRowFilter implements RowFilterInterface {
private Pattern rowKeyPattern = null;
private String rowKeyRegExp = null;
@Deprecated
private Map<byte [], byte[]> equalsMap =
new TreeMap<byte [], byte[]>(Bytes.BYTES_COMPARATOR);
@Deprecated
private Set<byte []> nullColumns =
new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RegExpRowFilter() {
super();
}
/**
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
*/
public RegExpRowFilter(final String rowKeyRegExp) {
this.rowKeyRegExp = rowKeyRegExp;
}
/**
* @deprecated Column filtering has been replaced by {@link ColumnValueFilter}
* Constructor that takes a row key regular expression to filter on.
*
* @param rowKeyRegExp
* @param columnFilter
*/
@Deprecated
public RegExpRowFilter(final String rowKeyRegExp,
final Map<byte [], byte[]> columnFilter) {
this.rowKeyRegExp = rowKeyRegExp;
this.setColumnFilters(columnFilter);
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, byte [] rowKey) {
//doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/**
* @deprecated Column filtering has been replaced by {@link ColumnValueFilter}
* Specify a value that must be matched for the given column.
*
* @param colKey
* the column to match on
* @param value
* the value that must equal the stored value.
*/
@Deprecated
public void setColumnFilter(final byte [] colKey, final byte[] value) {
if (value == null) {
nullColumns.add(colKey);
} else {
equalsMap.put(colKey, value);
}
}
/**
* @deprecated Column filtering has been replaced by {@link ColumnValueFilter}
* Set column filters for a number of columns.
*
* @param columnFilter
* Map of columns with value criteria.
*/
@Deprecated
public void setColumnFilters(final Map<byte [], byte[]> columnFilter) {
if (null == columnFilter) {
nullColumns.clear();
equalsMap.clear();
} else {
for (Entry<byte [], byte[]> entry : columnFilter.entrySet()) {
setColumnFilter(entry.getKey(), entry.getValue());
}
}
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/**
*
* {@inheritDoc}
*/
public boolean filterAllRemaining() {
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRowKey(final byte [] rowKey) {
return (filtersByRowKey() && rowKey != null)?
!getRowKeyPattern().matcher(Bytes.toString(rowKey)).matches():
false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterColumn(final byte [] rowKey, final byte [] colKey,
final byte[] data) {
if (filterRowKey(rowKey)) {
return true;
}
if (filtersByColumnValue()) {
byte[] filterValue = equalsMap.get(colKey);
if (null != filterValue) {
return !Arrays.equals(filterValue, data);
}
}
if (nullColumns.contains(colKey)) {
if (data != null && !HLogEdit.isDeleted(data)) {
return true;
}
}
return false;
}
/**
*
* {@inheritDoc}
*/
public boolean filterRow(final SortedMap<byte [], byte[]> columns) {
for (Entry<byte [], byte[]> col : columns.entrySet()) {
if (nullColumns.contains(col.getKey())
&& !HLogEdit.isDeleted(col.getValue())) {
return true;
}
}
for (byte [] col : equalsMap.keySet()) {
if (!columns.containsKey(col)) {
return true;
}
}
return false;
}
@Deprecated
private boolean filtersByColumnValue() {
return equalsMap != null && equalsMap.size() > 0;
}
private boolean filtersByRowKey() {
return null != rowKeyPattern || null != rowKeyRegExp;
}
private String getRowKeyRegExp() {
if (null == rowKeyRegExp && rowKeyPattern != null) {
rowKeyRegExp = rowKeyPattern.toString();
}
return rowKeyRegExp;
}
private Pattern getRowKeyPattern() {
if (rowKeyPattern == null && rowKeyRegExp != null) {
rowKeyPattern = Pattern.compile(rowKeyRegExp);
}
return rowKeyPattern;
}
/**
*
* {@inheritDoc}
*/
public void readFields(final DataInput in) throws IOException {
boolean hasRowKeyPattern = in.readBoolean();
if (hasRowKeyPattern) {
rowKeyRegExp = in.readUTF();
}
// equals map
equalsMap.clear();
int size = in.readInt();
for (int i = 0; i < size; i++) {
byte [] key = Bytes.readByteArray(in);
int len = in.readInt();
byte[] value = null;
if (len >= 0) {
value = new byte[len];
in.readFully(value);
}
setColumnFilter(key, value);
}
// nullColumns
nullColumns.clear();
size = in.readInt();
for (int i = 0; i < size; i++) {
setColumnFilter(Bytes.readByteArray(in), null);
}
}
/**
*
* {@inheritDoc}
*/
public void validate(final byte [][] columns) {
Set<byte []> invalids = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
for (byte [] colKey : getFilterColumns()) {
boolean found = false;
for (byte [] col : columns) {
if (Bytes.equals(col, colKey)) {
found = true;
break;
}
}
if (!found) {
invalids.add(colKey);
}
}
if (invalids.size() > 0) {
throw new InvalidRowFilterException(String.format(
"RowFilter contains criteria on columns %s not in %s", invalids,
Arrays.toString(columns)));
}
}
@Deprecated
private Set<byte []> getFilterColumns() {
Set<byte []> cols = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
cols.addAll(equalsMap.keySet());
cols.addAll(nullColumns);
return cols;
}
/**
*
* {@inheritDoc}
*/
public void write(final DataOutput out) throws IOException {
if (!filtersByRowKey()) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeUTF(getRowKeyRegExp());
}
// equalsMap
out.writeInt(equalsMap.size());
for (Entry<byte [], byte[]> entry : equalsMap.entrySet()) {
Bytes.writeByteArray(out, entry.getKey());
byte[] value = entry.getValue();
out.writeInt(value.length);
out.write(value);
}
// null columns
out.writeInt(nullColumns.size());
for (byte [] col : nullColumns) {
Bytes.writeByteArray(out, col);
}
}
}

228
src/java/org/apache/hadoop/hbase/filter/RowFilterInterface.java
View File

@ -1,115 +1,115 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.util.SortedMap;
import org.apache.hadoop.io.Writable;
/**
*
* Interface used for row-level filters applied to HRegion.HScanner scan
* results during calls to next().
*/
public interface RowFilterInterface extends Writable {
/**
* Resets the state of the filter. Used prior to the start of a Region scan.
*
*/
void reset();
/**
* Called to let filter know the final decision (to pass or filter) on a
* given row. With out HScanner calling this, the filter does not know if a
* row passed filtering even if it passed the row itself because other
* filters may have failed the row. E.g. when this filter is a member of a
* RowFilterSet with an OR operator.
*
* @see RowFilterSet
* @param filtered
* @param key
*/
void rowProcessed(boolean filtered, byte [] key);
/**
* Returns whether or not the filter should always be processed in any
* filtering call. This precaution is necessary for filters that maintain
* state and need to be updated according to their response to filtering
* calls (see WhileMatchRowFilter for an example). At times, filters nested
* in RowFilterSets may or may not be called because the RowFilterSet
* determines a result as fast as possible. Returning true for
* processAlways() ensures that the filter will always be called.
*
* @return whether or not to always process the filter
*/
boolean processAlways();
/**
* Determines if the filter has decided that all remaining results should be
* filtered (skipped). This is used to prevent the scanner from scanning a
* the rest of the HRegion when for sure the filter will exclude all
* remaining rows.
*
* @return true if the filter intends to filter all remaining rows.
*/
boolean filterAllRemaining();
/**
* Filters on just a row key. This is the first chance to stop a row.
*
* @param rowKey
* @return true if given row key is filtered and row should not be processed.
*/
boolean filterRowKey(final byte [] rowKey);
/**
* Filters on row key, column name, and column value. This will take individual columns out of a row,
* but the rest of the row will still get through.
*
* @param rowKey row key to filter on.
* @param colunmName column name to filter on
* @param columnValue column value to filter on
* @return true if row filtered and should not be processed.
*/
boolean filterColumn(final byte [] rowKey, final byte [] colunmName,
final byte[] columnValue);
/**
* Filter on the fully assembled row. This is the last chance to stop a row.
*
* @param columns
* @return true if row filtered and should not be processed.
*/
boolean filterRow(final SortedMap<byte [], byte[]> columns);
/**
* Validates that this filter applies only to a subset of the given columns.
* This check is done prior to opening of scanner due to the limitation that
* filtering of columns is dependent on the retrieval of those columns within
* the HRegion. Criteria on columns that are not part of a scanner's column
* list will be ignored. In the case of null value filters, all rows will pass
* the filter. This behavior should be 'undefined' for the user and therefore
* not permitted.
*
* @param columns
*/
void validate(final byte [][] columns);
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.util.SortedMap;
import org.apache.hadoop.io.Writable;
/**
*
* Interface used for row-level filters applied to HRegion.HScanner scan
* results during calls to next().
*/
public interface RowFilterInterface extends Writable {
/**
* Resets the state of the filter. Used prior to the start of a Region scan.
*
*/
void reset();
/**
* Called to let filter know the final decision (to pass or filter) on a
* given row. With out HScanner calling this, the filter does not know if a
* row passed filtering even if it passed the row itself because other
* filters may have failed the row. E.g. when this filter is a member of a
* RowFilterSet with an OR operator.
*
* @see RowFilterSet
* @param filtered
* @param key
*/
void rowProcessed(boolean filtered, byte [] key);
/**
* Returns whether or not the filter should always be processed in any
* filtering call. This precaution is necessary for filters that maintain
* state and need to be updated according to their response to filtering
* calls (see WhileMatchRowFilter for an example). At times, filters nested
* in RowFilterSets may or may not be called because the RowFilterSet
* determines a result as fast as possible. Returning true for
* processAlways() ensures that the filter will always be called.
*
* @return whether or not to always process the filter
*/
boolean processAlways();
/**
* Determines if the filter has decided that all remaining results should be
* filtered (skipped). This is used to prevent the scanner from scanning a
* the rest of the HRegion when for sure the filter will exclude all
* remaining rows.
*
* @return true if the filter intends to filter all remaining rows.
*/
boolean filterAllRemaining();
/**
* Filters on just a row key. This is the first chance to stop a row.
*
* @param rowKey
* @return true if given row key is filtered and row should not be processed.
*/
boolean filterRowKey(final byte [] rowKey);
/**
* Filters on row key, column name, and column value. This will take individual columns out of a row,
* but the rest of the row will still get through.
*
* @param rowKey row key to filter on.
* @param colunmName column name to filter on
* @param columnValue column value to filter on
* @return true if row filtered and should not be processed.
*/
boolean filterColumn(final byte [] rowKey, final byte [] colunmName,
final byte[] columnValue);
/**
* Filter on the fully assembled row. This is the last chance to stop a row.
*
* @param columns
* @return true if row filtered and should not be processed.
*/
boolean filterRow(final SortedMap<byte [], byte[]> columns);
/**
* Validates that this filter applies only to a subset of the given columns.
* This check is done prior to opening of scanner due to the limitation that
* filtering of columns is dependent on the retrieval of those columns within
* the HRegion. Criteria on columns that are not part of a scanner's column
* list will be ignored. In the case of null value filters, all rows will pass
* the filter. This behavior should be 'undefined' for the user and therefore
* not permitted.
*
* @param columns
*/
void validate(final byte [][] columns);
}

462
src/java/org/apache/hadoop/hbase/filter/RowFilterSet.java
View File

@ -1,231 +1,231 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.io.ObjectWritable;
/**
* Implementation of RowFilterInterface that represents a set of RowFilters
* which will be evaluated with a specified boolean operator MUST_PASS_ALL
* (!AND) or MUST_PASS_ONE (!OR). Since you can use RowFilterSets as children
* of RowFilterSet, you can create a hierarchy of filters to be evaluated.
*/
public class RowFilterSet implements RowFilterInterface {
/** set operator */
public static enum Operator {
/** !AND */
MUST_PASS_ALL,
/** !OR */
MUST_PASS_ONE
}
private Operator operator = Operator.MUST_PASS_ALL;
private Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RowFilterSet() {
super();
}
/**
* Constructor that takes a set of RowFilters. The default operator
* MUST_PASS_ALL is assumed.
*
* @param rowFilters
*/
public RowFilterSet(final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
}
/**
* Constructor that takes a set of RowFilters and an operator.
*
* @param operator Operator to process filter set with.
* @param rowFilters Set of row filters.
*/
public RowFilterSet(final Operator operator,
final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
this.operator = operator;
}
/** {@inheritDoc} */
public void validate(final byte [][] columns) {
for (RowFilterInterface filter : filters) {
filter.validate(columns);
}
}
/** {@inheritDoc} */
public void reset() {
for (RowFilterInterface filter : filters) {
filter.reset();
}
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, byte [] rowKey) {
for (RowFilterInterface filter : filters) {
filter.rowProcessed(filtered, rowKey);
}
}
/** {@inheritDoc} */
public boolean processAlways() {
for (RowFilterInterface filter : filters) {
if (filter.processAlways()) {
return true;
}
}
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining()) {
return true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining()) {
return false;
}
}
}
return result;
}
/** {@inheritDoc} */
public boolean filterRowKey(final byte [] rowKey) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterRowKey(rowKey)) {
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey)) {
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterRowKey(rowKey);
}
}
return result;
}
/** {@inheritDoc} */
public boolean filterColumn(final byte [] rowKey, final byte [] colKey,
final byte[] data) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() ||
filter.filterColumn(rowKey, colKey, data)) {
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() &&
!filter.filterColumn(rowKey, colKey, data)) {
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterColumn(rowKey, colKey, data);
}
}
return result;
}
/** {@inheritDoc} */
public boolean filterRow(final SortedMap<byte [], byte[]> columns) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterRow(columns)) {
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filterRow(columns)) {
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterRow(columns);
}
}
return result;
}
/** {@inheritDoc} */
public void readFields(final DataInput in) throws IOException {
Configuration conf = new HBaseConfiguration();
byte opByte = in.readByte();
operator = Operator.values()[opByte];
int size = in.readInt();
if (size > 0) {
filters = new HashSet<RowFilterInterface>();
for (int i = 0; i < size; i++) {
RowFilterInterface filter = (RowFilterInterface) ObjectWritable
.readObject(in, conf);
filters.add(filter);
}
}
}
/** {@inheritDoc} */
public void write(final DataOutput out) throws IOException {
Configuration conf = new HBaseConfiguration();
out.writeByte(operator.ordinal());
out.writeInt(filters.size());
for (RowFilterInterface filter : filters) {
ObjectWritable.writeObject(out, filter, RowFilterInterface.class, conf);
}
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.io.ObjectWritable;
/**
* Implementation of RowFilterInterface that represents a set of RowFilters
* which will be evaluated with a specified boolean operator MUST_PASS_ALL
* (!AND) or MUST_PASS_ONE (!OR). Since you can use RowFilterSets as children
* of RowFilterSet, you can create a hierarchy of filters to be evaluated.
*/
public class RowFilterSet implements RowFilterInterface {
/** set operator */
public static enum Operator {
/** !AND */
MUST_PASS_ALL,
/** !OR */
MUST_PASS_ONE
}
private Operator operator = Operator.MUST_PASS_ALL;
private Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public RowFilterSet() {
super();
}
/**
* Constructor that takes a set of RowFilters. The default operator
* MUST_PASS_ALL is assumed.
*
* @param rowFilters
*/
public RowFilterSet(final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
}
/**
* Constructor that takes a set of RowFilters and an operator.
*
* @param operator Operator to process filter set with.
* @param rowFilters Set of row filters.
*/
public RowFilterSet(final Operator operator,
final Set<RowFilterInterface> rowFilters) {
this.filters = rowFilters;
this.operator = operator;
}
/** {@inheritDoc} */
public void validate(final byte [][] columns) {
for (RowFilterInterface filter : filters) {
filter.validate(columns);
}
}
/** {@inheritDoc} */
public void reset() {
for (RowFilterInterface filter : filters) {
filter.reset();
}
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, byte [] rowKey) {
for (RowFilterInterface filter : filters) {
filter.rowProcessed(filtered, rowKey);
}
}
/** {@inheritDoc} */
public boolean processAlways() {
for (RowFilterInterface filter : filters) {
if (filter.processAlways()) {
return true;
}
}
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining()) {
return true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining()) {
return false;
}
}
}
return result;
}
/** {@inheritDoc} */
public boolean filterRowKey(final byte [] rowKey) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterRowKey(rowKey)) {
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey)) {
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterRowKey(rowKey);
}
}
return result;
}
/** {@inheritDoc} */
public boolean filterColumn(final byte [] rowKey, final byte [] colKey,
final byte[] data) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() ||
filter.filterColumn(rowKey, colKey, data)) {
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() &&
!filter.filterColumn(rowKey, colKey, data)) {
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterColumn(rowKey, colKey, data);
}
}
return result;
}
/** {@inheritDoc} */
public boolean filterRow(final SortedMap<byte [], byte[]> columns) {
boolean resultFound = false;
boolean result = operator == Operator.MUST_PASS_ONE;
for (RowFilterInterface filter : filters) {
if (!resultFound) {
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterRow(columns)) {
result = true;
resultFound = true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
if (!filter.filterAllRemaining() && !filter.filterRow(columns)) {
result = false;
resultFound = true;
}
}
} else if (filter.processAlways()) {
filter.filterRow(columns);
}
}
return result;
}
/** {@inheritDoc} */
public void readFields(final DataInput in) throws IOException {
Configuration conf = new HBaseConfiguration();
byte opByte = in.readByte();
operator = Operator.values()[opByte];
int size = in.readInt();
if (size > 0) {
filters = new HashSet<RowFilterInterface>();
for (int i = 0; i < size; i++) {
RowFilterInterface filter = (RowFilterInterface) ObjectWritable
.readObject(in, conf);
filters.add(filter);
}
}
}
/** {@inheritDoc} */
public void write(final DataOutput out) throws IOException {
Configuration conf = new HBaseConfiguration();
out.writeByte(operator.ordinal());
out.writeInt(filters.size());
for (RowFilterInterface filter : filters) {
ObjectWritable.writeObject(out, filter, RowFilterInterface.class, conf);
}
}
}

276
src/java/org/apache/hadoop/hbase/filter/StopRowFilter.java
View File

@ -1,139 +1,139 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Implementation of RowFilterInterface that filters out rows greater than or
* equal to a specified rowKey.
*/
public class StopRowFilter implements RowFilterInterface {
private byte [] stopRowKey;
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public StopRowFilter() {
super();
}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public StopRowFilter(final byte [] stopRowKey) {
this.stopRowKey = stopRowKey;
}
/**
* An accessor for the stopRowKey
*
* @return the filter's stopRowKey
*/
public byte [] getStopRowKey() {
return this.stopRowKey;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final byte [][] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, byte [] rowKey) {
// Doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
return false;
}
/** {@inheritDoc} */
public boolean filterRowKey(final byte [] rowKey) {
if (rowKey == null) {
if (this.stopRowKey == null) {
return true;
}
return false;
}
return Bytes.compareTo(stopRowKey, rowKey) <= 0;
}
/**
* {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling the rowKey-only version of filter.
*/
public boolean filterColumn(@SuppressWarnings("unused") final byte [] rowKey,
@SuppressWarnings("unused") final byte [] colKey,
@SuppressWarnings("unused") final byte[] data) {
return filterRowKey(rowKey);
}
/** {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling filterAllRemaining().
*
* @param columns
*/
public boolean filterRow(@SuppressWarnings("unused")
final SortedMap<byte [], byte[]> columns) {
return filterAllRemaining();
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
this.stopRowKey = Bytes.readByteArray(in);
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
Bytes.writeByteArray(out, this.stopRowKey);
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Implementation of RowFilterInterface that filters out rows greater than or
* equal to a specified rowKey.
*/
public class StopRowFilter implements RowFilterInterface {
private byte [] stopRowKey;
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public StopRowFilter() {
super();
}
/**
* Constructor that takes a stopRowKey on which to filter
*
* @param stopRowKey rowKey to filter on.
*/
public StopRowFilter(final byte [] stopRowKey) {
this.stopRowKey = stopRowKey;
}
/**
* An accessor for the stopRowKey
*
* @return the filter's stopRowKey
*/
public byte [] getStopRowKey() {
return this.stopRowKey;
}
/**
*
* {@inheritDoc}
*/
public void validate(@SuppressWarnings("unused") final byte [][] columns) {
// Doesn't filter columns
}
/**
*
* {@inheritDoc}
*/
public void reset() {
// Nothing to reset
}
/** {@inheritDoc} */
@SuppressWarnings("unused")
public void rowProcessed(boolean filtered, byte [] rowKey) {
// Doesn't care
}
/** {@inheritDoc} */
public boolean processAlways() {
return false;
}
/** {@inheritDoc} */
public boolean filterAllRemaining() {
return false;
}
/** {@inheritDoc} */
public boolean filterRowKey(final byte [] rowKey) {
if (rowKey == null) {
if (this.stopRowKey == null) {
return true;
}
return false;
}
return Bytes.compareTo(stopRowKey, rowKey) <= 0;
}
/**
* {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling the rowKey-only version of filter.
*/
public boolean filterColumn(@SuppressWarnings("unused") final byte [] rowKey,
@SuppressWarnings("unused") final byte [] colKey,
@SuppressWarnings("unused") final byte[] data) {
return filterRowKey(rowKey);
}
/** {@inheritDoc}
*
* Because StopRowFilter does not examine column information, this method
* defaults to calling filterAllRemaining().
*
* @param columns
*/
public boolean filterRow(@SuppressWarnings("unused")
final SortedMap<byte [], byte[]> columns) {
return filterAllRemaining();
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
this.stopRowKey = Bytes.readByteArray(in);
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
Bytes.writeByteArray(out, this.stopRowKey);
}
}

302
src/java/org/apache/hadoop/hbase/filter/WhileMatchRowFilter.java
View File

@ -1,151 +1,151 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
/**
* WhileMatchRowFilter is a wrapper filter that filters everything after the
* first filtered row. Once the nested filter returns true for either of it's
* filter(..) methods or filterNotNull(SortedMap<Text, byte[]>), this wrapper's
* filterAllRemaining() will return true. All filtering methods will
* thereafter defer to the result of filterAllRemaining().
*/
public class WhileMatchRowFilter implements RowFilterInterface {
private boolean filterAllRemaining = false;
private RowFilterInterface filter;
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public WhileMatchRowFilter() {
super();
}
/**
* Constructor
* @param filter
*/
public WhileMatchRowFilter(RowFilterInterface filter) {
this.filter = filter;
}
/**
* Returns the internal filter being wrapped
*
* @return the internal filter
*/
public RowFilterInterface getInternalFilter() {
return this.filter;
}
/** {@inheritDoc} */
public void reset() {
this.filterAllRemaining = false;
this.filter.reset();
}
/** {@inheritDoc} */
public boolean processAlways() {
return true;
}
/**
* Returns true once the nested filter has filtered out a row (returned true
* on a call to one of it's filtering methods). Until then it returns false.
*
* @return true/false whether the nested filter has returned true on a filter
* call.
*/
public boolean filterAllRemaining() {
return this.filterAllRemaining || this.filter.filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filterRowKey(final byte [] rowKey) {
changeFAR(this.filter.filterRowKey(rowKey));
return filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filterColumn(final byte [] rowKey, final byte [] colKey,
final byte[] data) {
changeFAR(this.filter.filterColumn(rowKey, colKey, data));
return filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filterRow(final SortedMap<byte [], byte[]> columns) {
changeFAR(this.filter.filterRow(columns));
return filterAllRemaining();
}
/**
* Change filterAllRemaining from false to true if value is true, otherwise
* leave as is.
*
* @param value
*/
private void changeFAR(boolean value) {
this.filterAllRemaining = this.filterAllRemaining || value;
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, byte [] rowKey) {
this.filter.rowProcessed(filtered, rowKey);
}
/** {@inheritDoc} */
public void validate(final byte [][] columns) {
this.filter.validate(columns);
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
String className = in.readUTF();
try {
this.filter = (RowFilterInterface)(Class.forName(className).
newInstance());
this.filter.readFields(in);
} catch (InstantiationException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (IllegalAccessException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (ClassNotFoundException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
}
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
out.writeUTF(this.filter.getClass().getName());
this.filter.write(out);
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.SortedMap;
/**
* WhileMatchRowFilter is a wrapper filter that filters everything after the
* first filtered row. Once the nested filter returns true for either of it's
* filter(..) methods or filterNotNull(SortedMap<Text, byte[]>), this wrapper's
* filterAllRemaining() will return true. All filtering methods will
* thereafter defer to the result of filterAllRemaining().
*/
public class WhileMatchRowFilter implements RowFilterInterface {
private boolean filterAllRemaining = false;
private RowFilterInterface filter;
/**
* Default constructor, filters nothing. Required though for RPC
* deserialization.
*/
public WhileMatchRowFilter() {
super();
}
/**
* Constructor
* @param filter
*/
public WhileMatchRowFilter(RowFilterInterface filter) {
this.filter = filter;
}
/**
* Returns the internal filter being wrapped
*
* @return the internal filter
*/
public RowFilterInterface getInternalFilter() {
return this.filter;
}
/** {@inheritDoc} */
public void reset() {
this.filterAllRemaining = false;
this.filter.reset();
}
/** {@inheritDoc} */
public boolean processAlways() {
return true;
}
/**
* Returns true once the nested filter has filtered out a row (returned true
* on a call to one of it's filtering methods). Until then it returns false.
*
* @return true/false whether the nested filter has returned true on a filter
* call.
*/
public boolean filterAllRemaining() {
return this.filterAllRemaining || this.filter.filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filterRowKey(final byte [] rowKey) {
changeFAR(this.filter.filterRowKey(rowKey));
return filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filterColumn(final byte [] rowKey, final byte [] colKey,
final byte[] data) {
changeFAR(this.filter.filterColumn(rowKey, colKey, data));
return filterAllRemaining();
}
/** {@inheritDoc} */
public boolean filterRow(final SortedMap<byte [], byte[]> columns) {
changeFAR(this.filter.filterRow(columns));
return filterAllRemaining();
}
/**
* Change filterAllRemaining from false to true if value is true, otherwise
* leave as is.
*
* @param value
*/
private void changeFAR(boolean value) {
this.filterAllRemaining = this.filterAllRemaining || value;
}
/** {@inheritDoc} */
public void rowProcessed(boolean filtered, byte [] rowKey) {
this.filter.rowProcessed(filtered, rowKey);
}
/** {@inheritDoc} */
public void validate(final byte [][] columns) {
this.filter.validate(columns);
}
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
String className = in.readUTF();
try {
this.filter = (RowFilterInterface)(Class.forName(className).
newInstance());
this.filter.readFields(in);
} catch (InstantiationException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (IllegalAccessException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
} catch (ClassNotFoundException e) {
throw new RuntimeException("Failed to deserialize WhileMatchRowFilter.",
e);
}
}
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
out.writeUTF(this.filter.getClass().getName());
this.filter.write(out);
}
}

222
src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
View File

@ -1,111 +1,111 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.mapred;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
/**
* Convert Map/Reduce output and write it to an HBase table
*/
public class TableOutputFormat extends
FileOutputFormat<ImmutableBytesWritable, BatchUpdate> {
/** JobConf parameter that specifies the output table */
public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
/**
* Convert Reduce output (key, value) to (HStoreKey, KeyedDataArrayWritable)
* and write to an HBase table
*/
protected class TableRecordWriter
implements RecordWriter<ImmutableBytesWritable, BatchUpdate> {
private HTable m_table;
/**
* Instantiate a TableRecordWriter with the HBase HClient for writing.
*
* @param table
*/
public TableRecordWriter(HTable table) {
m_table = table;
}
/** {@inheritDoc} */
public void close(@SuppressWarnings("unused") Reporter reporter) {
// Nothing to do.
}
/** {@inheritDoc} */
public void write(@SuppressWarnings("unused") ImmutableBytesWritable key,
BatchUpdate value) throws IOException {
m_table.commit(value);
}
}
/** {@inheritDoc} */
@Override
@SuppressWarnings("unchecked")
public RecordWriter getRecordWriter(
@SuppressWarnings("unused") FileSystem ignored,
JobConf job,
@SuppressWarnings("unused") String name,
@SuppressWarnings("unused") Progressable progress) throws IOException {
// expecting exactly one path
String tableName = job.get(OUTPUT_TABLE);
HTable table = null;
try {
table = new HTable(new HBaseConfiguration(job), tableName);
} catch(IOException e) {
LOG.error(e);
throw e;
}
return new TableRecordWriter(table);
}
/** {@inheritDoc} */
@Override
@SuppressWarnings("unused")
public void checkOutputSpecs(FileSystem ignored, JobConf job)
throws FileAlreadyExistsException, InvalidJobConfException, IOException {
String tableName = job.get(OUTPUT_TABLE);
if(tableName == null) {
throw new IOException("Must specify table name");
}
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.mapred;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
/**
* Convert Map/Reduce output and write it to an HBase table
*/
public class TableOutputFormat extends
FileOutputFormat<ImmutableBytesWritable, BatchUpdate> {
/** JobConf parameter that specifies the output table */
public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
/**
* Convert Reduce output (key, value) to (HStoreKey, KeyedDataArrayWritable)
* and write to an HBase table
*/
protected class TableRecordWriter
implements RecordWriter<ImmutableBytesWritable, BatchUpdate> {
private HTable m_table;
/**
* Instantiate a TableRecordWriter with the HBase HClient for writing.
*
* @param table
*/
public TableRecordWriter(HTable table) {
m_table = table;
}
/** {@inheritDoc} */
public void close(@SuppressWarnings("unused") Reporter reporter) {
// Nothing to do.
}
/** {@inheritDoc} */
public void write(@SuppressWarnings("unused") ImmutableBytesWritable key,
BatchUpdate value) throws IOException {
m_table.commit(value);
}
}
/** {@inheritDoc} */
@Override
@SuppressWarnings("unchecked")
public RecordWriter getRecordWriter(
@SuppressWarnings("unused") FileSystem ignored,
JobConf job,
@SuppressWarnings("unused") String name,
@SuppressWarnings("unused") Progressable progress) throws IOException {
// expecting exactly one path
String tableName = job.get(OUTPUT_TABLE);
HTable table = null;
try {
table = new HTable(new HBaseConfiguration(job), tableName);
} catch(IOException e) {
LOG.error(e);
throw e;
}
return new TableRecordWriter(table);
}
/** {@inheritDoc} */
@Override
@SuppressWarnings("unused")
public void checkOutputSpecs(FileSystem ignored, JobConf job)
throws FileAlreadyExistsException, InvalidJobConfException, IOException {
String tableName = job.get(OUTPUT_TABLE);
if(tableName == null) {
throw new IOException("Must specify table name");
}
}
}

546
src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java
View File

@ -1,273 +1,273 @@
/**
* Copyright 2008 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Scanner scans both the memcache and the HStore
*/
class HStoreScanner implements InternalScanner {
static final Log LOG = LogFactory.getLog(HStoreScanner.class);
private InternalScanner[] scanners;
private TreeMap<byte [], byte []>[] resultSets;
private HStoreKey[] keys;
private boolean wildcardMatch = false;
private boolean multipleMatchers = false;
private RowFilterInterface dataFilter;
private HStore store;
/** Create an Scanner with a handle on the memcache and HStore files. */
@SuppressWarnings("unchecked")
HStoreScanner(HStore store, byte [][] targetCols, byte [] firstRow,
long timestamp, RowFilterInterface filter)
throws IOException {
this.store = store;
this.dataFilter = filter;
if (null != dataFilter) {
dataFilter.reset();
}
this.scanners = new InternalScanner[2];
this.resultSets = new TreeMap[scanners.length];
this.keys = new HStoreKey[scanners.length];
try {
scanners[0] = store.memcache.getScanner(timestamp, targetCols, firstRow);
scanners[1] = new StoreFileScanner(store, timestamp, targetCols, firstRow);
for (int i = 0; i < scanners.length; i++) {
if (scanners[i].isWildcardScanner()) {
this.wildcardMatch = true;
}
if (scanners[i].isMultipleMatchScanner()) {
this.multipleMatchers = true;
}
}
} catch(IOException e) {
for (int i = 0; i < this.scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
throw e;
}
// Advance to the first key in each scanner.
// All results will match the required column-set and scanTime.
for (int i = 0; i < scanners.length; i++) {
keys[i] = new HStoreKey();
resultSets[i] = new TreeMap<byte [], byte []>(Bytes.BYTES_COMPARATOR);
if(scanners[i] != null && !scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
/** @return true if the scanner is a wild card scanner */
public boolean isWildcardScanner() {
return wildcardMatch;
}
/** @return true if the scanner is a multiple match scanner */
public boolean isMultipleMatchScanner() {
return multipleMatchers;
}
/** {@inheritDoc} */
public boolean next(HStoreKey key, SortedMap<byte [], byte[]> results)
throws IOException {
// Filtered flag is set by filters. If a cell has been 'filtered out'
// -- i.e. it is not to be returned to the caller -- the flag is 'true'.
boolean filtered = true;
boolean moreToFollow = true;
while (filtered && moreToFollow) {
// Find the lowest-possible key.
byte [] chosenRow = null;
long chosenTimestamp = -1;
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
(chosenRow == null ||
(Bytes.compareTo(keys[i].getRow(), chosenRow) < 0) ||
((Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = keys[i].getRow();
chosenTimestamp = keys[i].getTimestamp();
}
}
// Filter whole row by row key?
filtered = dataFilter != null? dataFilter.filterRowKey(chosenRow) : false;
// Store the key and results for each sub-scanner. Merge them as
// appropriate.
if (chosenTimestamp >= 0 && !filtered) {
// Here we are setting the passed in key with current row+timestamp
key.setRow(chosenRow);
key.setVersion(chosenTimestamp);
key.setColumn(HConstants.EMPTY_BYTE_ARRAY);
// Keep list of deleted cell keys within this row. We need this
// because as we go through scanners, the delete record may be in an
// early scanner and then the same record with a non-delete, non-null
// value in a later. Without history of what we've seen, we'll return
// deleted values. This List should not ever grow too large since we
// are only keeping rows and columns that match those set on the
// scanner and which have delete values. If memory usage becomes a
// problem, could redo as bloom filter.
List<HStoreKey> deletes = new ArrayList<HStoreKey>();
for (int i = 0; i < scanners.length && !filtered; i++) {
while ((scanners[i] != null
&& !filtered
&& moreToFollow)
&& (Bytes.compareTo(keys[i].getRow(), chosenRow) == 0)) {
// If we are doing a wild card match or there are multiple
// matchers per column, we need to scan all the older versions of
// this row to pick up the rest of the family members
if (!wildcardMatch
&& !multipleMatchers
&& (keys[i].getTimestamp() != chosenTimestamp)) {
break;
}
// NOTE: We used to do results.putAll(resultSets[i]);
// but this had the effect of overwriting newer
// values with older ones. So now we only insert
// a result if the map does not contain the key.
HStoreKey hsk = new HStoreKey(key.getRow(), HConstants.EMPTY_BYTE_ARRAY,
key.getTimestamp());
for (Map.Entry<byte [], byte[]> e : resultSets[i].entrySet()) {
hsk.setColumn(e.getKey());
if (HLogEdit.isDeleted(e.getValue())) {
if (!deletes.contains(hsk)) {
// Key changes as we cycle the for loop so add a copy to
// the set of deletes.
deletes.add(new HStoreKey(hsk));
}
} else if (!deletes.contains(hsk) &&
!filtered &&
moreToFollow &&
!results.containsKey(e.getKey())) {
if (dataFilter != null) {
// Filter whole row by column data?
filtered =
dataFilter.filterColumn(chosenRow, e.getKey(), e.getValue());
if (filtered) {
results.clear();
break;
}
}
results.put(e.getKey(), e.getValue());
}
}
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
}
for (int i = 0; i < scanners.length; i++) {
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
(Bytes.compareTo(keys[i].getRow(), chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
moreToFollow = chosenTimestamp >= 0;
if (dataFilter != null) {
if (dataFilter.filterAllRemaining()) {
moreToFollow = false;
}
}
if (results.size() <= 0 && !filtered) {
// There were no results found for this row. Marked it as
// 'filtered'-out otherwise we will not move on to the next row.
filtered = true;
}
}
// If we got no results, then there is no more to follow.
if (results == null || results.size() <= 0) {
moreToFollow = false;
}
// Make sure scanners closed if no more results
if (!moreToFollow) {
for (int i = 0; i < scanners.length; i++) {
if (null != scanners[i]) {
closeScanner(i);
}
}
}
return moreToFollow;
}
/** Shut down a single scanner */
void closeScanner(int i) {
try {
try {
scanners[i].close();
} catch (IOException e) {
LOG.warn(store.storeName + " failed closing scanner " + i, e);
}
} finally {
scanners[i] = null;
keys[i] = null;
resultSets[i] = null;
}
}
/** {@inheritDoc} */
public void close() {
for(int i = 0; i < scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
}
public Iterator<Map.Entry<HStoreKey, SortedMap<byte [], byte[]>>> iterator() {
throw new UnsupportedOperationException("Unimplemented serverside. " +
"next(HStoreKey, StortedMap(...) is more efficient");
}
}
/**
* Copyright 2008 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Scanner scans both the memcache and the HStore
*/
class HStoreScanner implements InternalScanner {
static final Log LOG = LogFactory.getLog(HStoreScanner.class);
private InternalScanner[] scanners;
private TreeMap<byte [], byte []>[] resultSets;
private HStoreKey[] keys;
private boolean wildcardMatch = false;
private boolean multipleMatchers = false;
private RowFilterInterface dataFilter;
private HStore store;
/** Create an Scanner with a handle on the memcache and HStore files. */
@SuppressWarnings("unchecked")
HStoreScanner(HStore store, byte [][] targetCols, byte [] firstRow,
long timestamp, RowFilterInterface filter)
throws IOException {
this.store = store;
this.dataFilter = filter;
if (null != dataFilter) {
dataFilter.reset();
}
this.scanners = new InternalScanner[2];
this.resultSets = new TreeMap[scanners.length];
this.keys = new HStoreKey[scanners.length];
try {
scanners[0] = store.memcache.getScanner(timestamp, targetCols, firstRow);
scanners[1] = new StoreFileScanner(store, timestamp, targetCols, firstRow);
for (int i = 0; i < scanners.length; i++) {
if (scanners[i].isWildcardScanner()) {
this.wildcardMatch = true;
}
if (scanners[i].isMultipleMatchScanner()) {
this.multipleMatchers = true;
}
}
} catch(IOException e) {
for (int i = 0; i < this.scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
throw e;
}
// Advance to the first key in each scanner.
// All results will match the required column-set and scanTime.
for (int i = 0; i < scanners.length; i++) {
keys[i] = new HStoreKey();
resultSets[i] = new TreeMap<byte [], byte []>(Bytes.BYTES_COMPARATOR);
if(scanners[i] != null && !scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
/** @return true if the scanner is a wild card scanner */
public boolean isWildcardScanner() {
return wildcardMatch;
}
/** @return true if the scanner is a multiple match scanner */
public boolean isMultipleMatchScanner() {
return multipleMatchers;
}
/** {@inheritDoc} */
public boolean next(HStoreKey key, SortedMap<byte [], byte[]> results)
throws IOException {
// Filtered flag is set by filters. If a cell has been 'filtered out'
// -- i.e. it is not to be returned to the caller -- the flag is 'true'.
boolean filtered = true;
boolean moreToFollow = true;
while (filtered && moreToFollow) {
// Find the lowest-possible key.
byte [] chosenRow = null;
long chosenTimestamp = -1;
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
(chosenRow == null ||
(Bytes.compareTo(keys[i].getRow(), chosenRow) < 0) ||
((Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = keys[i].getRow();
chosenTimestamp = keys[i].getTimestamp();
}
}
// Filter whole row by row key?
filtered = dataFilter != null? dataFilter.filterRowKey(chosenRow) : false;
// Store the key and results for each sub-scanner. Merge them as
// appropriate.
if (chosenTimestamp >= 0 && !filtered) {
// Here we are setting the passed in key with current row+timestamp
key.setRow(chosenRow);
key.setVersion(chosenTimestamp);
key.setColumn(HConstants.EMPTY_BYTE_ARRAY);
// Keep list of deleted cell keys within this row. We need this
// because as we go through scanners, the delete record may be in an
// early scanner and then the same record with a non-delete, non-null
// value in a later. Without history of what we've seen, we'll return
// deleted values. This List should not ever grow too large since we
// are only keeping rows and columns that match those set on the
// scanner and which have delete values. If memory usage becomes a
// problem, could redo as bloom filter.
List<HStoreKey> deletes = new ArrayList<HStoreKey>();
for (int i = 0; i < scanners.length && !filtered; i++) {
while ((scanners[i] != null
&& !filtered
&& moreToFollow)
&& (Bytes.compareTo(keys[i].getRow(), chosenRow) == 0)) {
// If we are doing a wild card match or there are multiple
// matchers per column, we need to scan all the older versions of
// this row to pick up the rest of the family members
if (!wildcardMatch
&& !multipleMatchers
&& (keys[i].getTimestamp() != chosenTimestamp)) {
break;
}
// NOTE: We used to do results.putAll(resultSets[i]);
// but this had the effect of overwriting newer
// values with older ones. So now we only insert
// a result if the map does not contain the key.
HStoreKey hsk = new HStoreKey(key.getRow(), HConstants.EMPTY_BYTE_ARRAY,
key.getTimestamp());
for (Map.Entry<byte [], byte[]> e : resultSets[i].entrySet()) {
hsk.setColumn(e.getKey());
if (HLogEdit.isDeleted(e.getValue())) {
if (!deletes.contains(hsk)) {
// Key changes as we cycle the for loop so add a copy to
// the set of deletes.
deletes.add(new HStoreKey(hsk));
}
} else if (!deletes.contains(hsk) &&
!filtered &&
moreToFollow &&
!results.containsKey(e.getKey())) {
if (dataFilter != null) {
// Filter whole row by column data?
filtered =
dataFilter.filterColumn(chosenRow, e.getKey(), e.getValue());
if (filtered) {
results.clear();
break;
}
}
results.put(e.getKey(), e.getValue());
}
}
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
}
for (int i = 0; i < scanners.length; i++) {
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
(Bytes.compareTo(keys[i].getRow(), chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
}
}
}
moreToFollow = chosenTimestamp >= 0;
if (dataFilter != null) {
if (dataFilter.filterAllRemaining()) {
moreToFollow = false;
}
}
if (results.size() <= 0 && !filtered) {
// There were no results found for this row. Marked it as
// 'filtered'-out otherwise we will not move on to the next row.
filtered = true;
}
}
// If we got no results, then there is no more to follow.
if (results == null || results.size() <= 0) {
moreToFollow = false;
}
// Make sure scanners closed if no more results
if (!moreToFollow) {
for (int i = 0; i < scanners.length; i++) {
if (null != scanners[i]) {
closeScanner(i);
}
}
}
return moreToFollow;
}
/** Shut down a single scanner */
void closeScanner(int i) {
try {
try {
scanners[i].close();
} catch (IOException e) {
LOG.warn(store.storeName + " failed closing scanner " + i, e);
}
} finally {
scanners[i] = null;
keys[i] = null;
resultSets[i] = null;
}
}
/** {@inheritDoc} */
public void close() {
for(int i = 0; i < scanners.length; i++) {
if(scanners[i] != null) {
closeScanner(i);
}
}
}
public Iterator<Map.Entry<HStoreKey, SortedMap<byte [], byte[]>>> iterator() {
throw new UnsupportedOperationException("Unimplemented serverside. " +
"next(HStoreKey, StortedMap(...) is more efficient");
}
}

186
src/test/org/apache/hadoop/hbase/filter/TestInclusiveStopRowFilter.java
View File

@ -1,93 +1,93 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests the inclusive stop row filter
*/
public class TestInclusiveStopRowFilter extends TestCase {
private final byte [] STOP_ROW = Bytes.toBytes("stop_row");
private final byte [] GOOD_ROW = Bytes.toBytes("good_row");
private final byte [] PAST_STOP_ROW = Bytes.toBytes("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new InclusiveStopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new InclusiveStopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filterRowKey(GOOD_ROW));
assertFalse("Filtering on " + STOP_ROW, filter.filterRowKey(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterRowKey(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filterColumn(GOOD_ROW, null,
null));
assertFalse("Filtering on " + STOP_ROW, filter.filterColumn(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterColumn(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterRow(null));
assertFalse("Filter a null", filter.filterRowKey(null));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests the inclusive stop row filter
*/
public class TestInclusiveStopRowFilter extends TestCase {
private final byte [] STOP_ROW = Bytes.toBytes("stop_row");
private final byte [] GOOD_ROW = Bytes.toBytes("good_row");
private final byte [] PAST_STOP_ROW = Bytes.toBytes("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new InclusiveStopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new InclusiveStopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filterRowKey(GOOD_ROW));
assertFalse("Filtering on " + STOP_ROW, filter.filterRowKey(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterRowKey(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filterColumn(GOOD_ROW, null,
null));
assertFalse("Filtering on " + STOP_ROW, filter.filterColumn(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterColumn(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterRow(null));
assertFalse("Filter a null", filter.filterRowKey(null));
}
}

198
src/test/org/apache/hadoop/hbase/filter/TestPageRowFilter.java
View File

@ -1,99 +1,99 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests for the page row filter
*/
public class TestPageRowFilter extends TestCase {
RowFilterInterface mainFilter;
final int ROW_LIMIT = 3;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new PageRowFilter(ROW_LIMIT);
}
/**
* test page size filter
* @throws Exception
*/
public void testPageSize() throws Exception {
pageSizeTests(mainFilter);
}
/**
* Test filter serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new PageRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
pageSizeTests(newFilter);
}
private void pageSizeTests(RowFilterInterface filter) throws Exception {
testFiltersBeyondPageSize(filter, ROW_LIMIT);
// Test reset works by going in again.
filter.reset();
testFiltersBeyondPageSize(filter, ROW_LIMIT);
}
private void testFiltersBeyondPageSize(final RowFilterInterface filter,
final int pageSize) {
for (int i = 0; i < (pageSize * 2); i++) {
byte [] row = Bytes.toBytes(Integer.toString(i));
boolean filterOut = filter.filterRowKey(row);
if (!filterOut) {
assertFalse("Disagrees with 'filter'", filter.filterAllRemaining());
} else {
// Once we have all for a page, calls to filterAllRemaining should
// stay true.
assertTrue("Disagrees with 'filter'", filter.filterAllRemaining());
assertTrue(i >= pageSize);
}
filter.rowProcessed(filterOut, row);
}
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests for the page row filter
*/
public class TestPageRowFilter extends TestCase {
RowFilterInterface mainFilter;
final int ROW_LIMIT = 3;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new PageRowFilter(ROW_LIMIT);
}
/**
* test page size filter
* @throws Exception
*/
public void testPageSize() throws Exception {
pageSizeTests(mainFilter);
}
/**
* Test filter serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new PageRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
pageSizeTests(newFilter);
}
private void pageSizeTests(RowFilterInterface filter) throws Exception {
testFiltersBeyondPageSize(filter, ROW_LIMIT);
// Test reset works by going in again.
filter.reset();
testFiltersBeyondPageSize(filter, ROW_LIMIT);
}
private void testFiltersBeyondPageSize(final RowFilterInterface filter,
final int pageSize) {
for (int i = 0; i < (pageSize * 2); i++) {
byte [] row = Bytes.toBytes(Integer.toString(i));
boolean filterOut = filter.filterRowKey(row);
if (!filterOut) {
assertFalse("Disagrees with 'filter'", filter.filterAllRemaining());
} else {
// Once we have all for a page, calls to filterAllRemaining should
// stay true.
assertTrue("Disagrees with 'filter'", filter.filterAllRemaining());
assertTrue(i >= pageSize);
}
filter.rowProcessed(filterOut, row);
}
}
}

392
src/test/org/apache/hadoop/hbase/filter/TestRegExpRowFilter.java
View File

@ -1,196 +1,196 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.TreeMap;
import junit.framework.TestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Tests for regular expression row filter
*/
public class TestRegExpRowFilter extends TestCase {
TreeMap<byte [], byte []> colvalues;
RowFilterInterface mainFilter;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
final String HOST_PREFIX = "org.apache.site-";
static byte [] GOOD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
this.colvalues = new TreeMap<byte [], byte[]>(Bytes.BYTES_COMPARATOR);
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(Bytes.toBytes(new String(new char [] {c})), GOOD_BYTES);
}
this.mainFilter = new RegExpRowFilter(HOST_PREFIX + ".*", colvalues);
}
/**
* Tests filtering using a regex on the row key
* @throws Exception
*/
public void testRegexOnRow() throws Exception {
regexRowTests(mainFilter);
}
/**
* Tests filtering using a regex on row and colum
* @throws Exception
*/
public void testRegexOnRowAndColumn() throws Exception {
regexRowColumnTests(mainFilter);
}
/**
* Only return values that are not null
* @throws Exception
*/
public void testFilterNotNull() throws Exception {
filterNotNullTests(mainFilter);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RegExpRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running all test.
regexRowTests(newFilter);
newFilter.reset();
regexRowColumnTests(newFilter);
newFilter.reset();
filterNotNullTests(newFilter);
}
private void regexRowTests(RowFilterInterface filter) throws Exception {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
byte [] t = createRow(c);
assertFalse("Failed with characer " + c, filter.filterRowKey(t));
}
String yahooSite = "com.yahoo.www";
assertTrue("Failed with character " +
yahooSite, filter.filterRowKey(Bytes.toBytes(yahooSite)));
}
private void regexRowColumnTests(RowFilterInterface filter)
throws UnsupportedEncodingException {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
byte [] t = createRow(c);
for (Map.Entry<byte [], byte []> e: this.colvalues.entrySet()) {
assertFalse("Failed on " + c,
filter.filterColumn(t, e.getKey(), e.getValue()));
}
}
// Try a row and column I know will pass.
char c = 'c';
byte [] r = createRow(c);
byte [] col = Bytes.toBytes(Character.toString(c));
assertFalse("Failed with character " + c,
filter.filterColumn(r, col, GOOD_BYTES));
// Do same but with bad bytes.
assertTrue("Failed with character " + c,
filter.filterColumn(r, col, "badbytes".getBytes(HConstants.UTF8_ENCODING)));
// Do with good bytes but bad column name. Should not filter out.
assertFalse("Failed with character " + c,
filter.filterColumn(r, Bytes.toBytes("badcolumn"), GOOD_BYTES));
// Good column, good bytes but bad row.
assertTrue("Failed with character " + c,
filter.filterColumn(Bytes.toBytes("bad row"),
Bytes.toBytes("badcolumn"), GOOD_BYTES));
}
private void filterNotNullTests(RowFilterInterface filter) throws Exception {
// Modify the filter to expect certain columns to be null:
// Expecting a row WITH columnKeys: a-d, WITHOUT columnKey: e
((RegExpRowFilter)filter).setColumnFilter(new byte [] {LAST_CHAR}, null);
char secondToLast = (char)(LAST_CHAR - 1);
char thirdToLast = (char)(LAST_CHAR - 2);
// Modify the row to be missing an expected columnKey (d)
colvalues.remove(new byte [] {(byte)secondToLast});
// Try a row that is missing an expected columnKey.
// Testing row with columnKeys: a-c
assertTrue("Failed with last columnKey " + thirdToLast, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys, and NO null-expected
// columnKeys.
// Testing row with columnKeys: a-d
colvalues.put(new byte [] {(byte)secondToLast}, GOOD_BYTES);
assertFalse("Failed with last columnKey " + secondToLast, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys AND a null-expected columnKey.
// Testing row with columnKeys: a-e
colvalues.put(new byte [] {LAST_CHAR}, GOOD_BYTES);
assertTrue("Failed with last columnKey " + LAST_CHAR, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys and a null-expected columnKey
// that maps to a null value.
// Testing row with columnKeys: a-e, e maps to null
colvalues.put(new byte [] {LAST_CHAR},
HLogEdit.deleteBytes.get());
assertFalse("Failed with last columnKey " + LAST_CHAR + " mapping to null.",
filter.filterRow(colvalues));
}
private byte [] createRow(final char c) {
return Bytes.toBytes(HOST_PREFIX + Character.toString(c));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.TreeMap;
import junit.framework.TestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.HLogEdit;
import org.apache.hadoop.hbase.util.Bytes;
/**
* Tests for regular expression row filter
*/
public class TestRegExpRowFilter extends TestCase {
TreeMap<byte [], byte []> colvalues;
RowFilterInterface mainFilter;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
final String HOST_PREFIX = "org.apache.site-";
static byte [] GOOD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
this.colvalues = new TreeMap<byte [], byte[]>(Bytes.BYTES_COMPARATOR);
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(Bytes.toBytes(new String(new char [] {c})), GOOD_BYTES);
}
this.mainFilter = new RegExpRowFilter(HOST_PREFIX + ".*", colvalues);
}
/**
* Tests filtering using a regex on the row key
* @throws Exception
*/
public void testRegexOnRow() throws Exception {
regexRowTests(mainFilter);
}
/**
* Tests filtering using a regex on row and colum
* @throws Exception
*/
public void testRegexOnRowAndColumn() throws Exception {
regexRowColumnTests(mainFilter);
}
/**
* Only return values that are not null
* @throws Exception
*/
public void testFilterNotNull() throws Exception {
filterNotNullTests(mainFilter);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RegExpRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running all test.
regexRowTests(newFilter);
newFilter.reset();
regexRowColumnTests(newFilter);
newFilter.reset();
filterNotNullTests(newFilter);
}
private void regexRowTests(RowFilterInterface filter) throws Exception {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
byte [] t = createRow(c);
assertFalse("Failed with characer " + c, filter.filterRowKey(t));
}
String yahooSite = "com.yahoo.www";
assertTrue("Failed with character " +
yahooSite, filter.filterRowKey(Bytes.toBytes(yahooSite)));
}
private void regexRowColumnTests(RowFilterInterface filter)
throws UnsupportedEncodingException {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
byte [] t = createRow(c);
for (Map.Entry<byte [], byte []> e: this.colvalues.entrySet()) {
assertFalse("Failed on " + c,
filter.filterColumn(t, e.getKey(), e.getValue()));
}
}
// Try a row and column I know will pass.
char c = 'c';
byte [] r = createRow(c);
byte [] col = Bytes.toBytes(Character.toString(c));
assertFalse("Failed with character " + c,
filter.filterColumn(r, col, GOOD_BYTES));
// Do same but with bad bytes.
assertTrue("Failed with character " + c,
filter.filterColumn(r, col, "badbytes".getBytes(HConstants.UTF8_ENCODING)));
// Do with good bytes but bad column name. Should not filter out.
assertFalse("Failed with character " + c,
filter.filterColumn(r, Bytes.toBytes("badcolumn"), GOOD_BYTES));
// Good column, good bytes but bad row.
assertTrue("Failed with character " + c,
filter.filterColumn(Bytes.toBytes("bad row"),
Bytes.toBytes("badcolumn"), GOOD_BYTES));
}
private void filterNotNullTests(RowFilterInterface filter) throws Exception {
// Modify the filter to expect certain columns to be null:
// Expecting a row WITH columnKeys: a-d, WITHOUT columnKey: e
((RegExpRowFilter)filter).setColumnFilter(new byte [] {LAST_CHAR}, null);
char secondToLast = (char)(LAST_CHAR - 1);
char thirdToLast = (char)(LAST_CHAR - 2);
// Modify the row to be missing an expected columnKey (d)
colvalues.remove(new byte [] {(byte)secondToLast});
// Try a row that is missing an expected columnKey.
// Testing row with columnKeys: a-c
assertTrue("Failed with last columnKey " + thirdToLast, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys, and NO null-expected
// columnKeys.
// Testing row with columnKeys: a-d
colvalues.put(new byte [] {(byte)secondToLast}, GOOD_BYTES);
assertFalse("Failed with last columnKey " + secondToLast, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys AND a null-expected columnKey.
// Testing row with columnKeys: a-e
colvalues.put(new byte [] {LAST_CHAR}, GOOD_BYTES);
assertTrue("Failed with last columnKey " + LAST_CHAR, filter.
filterRow(colvalues));
// Try a row that has all expected columnKeys and a null-expected columnKey
// that maps to a null value.
// Testing row with columnKeys: a-e, e maps to null
colvalues.put(new byte [] {LAST_CHAR},
HLogEdit.deleteBytes.get());
assertFalse("Failed with last columnKey " + LAST_CHAR + " mapping to null.",
filter.filterRow(colvalues));
}
private byte [] createRow(final char c) {
return Bytes.toBytes(HOST_PREFIX + Character.toString(c));
}
}

374
src/test/org/apache/hadoop/hbase/filter/TestRowFilterSet.java
View File

@ -1,187 +1,187 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;
import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests filter sets
*/
public class TestRowFilterSet extends TestCase {
RowFilterInterface filterMPALL;
RowFilterInterface filterMPONE;
static final int MAX_PAGES = 5;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
TreeMap<byte [], byte[]> colvalues;
static byte[] GOOD_BYTES = null;
static byte[] BAD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
BAD_BYTES = "def".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
colvalues = new TreeMap<byte [], byte[]>(Bytes.BYTES_COMPARATOR);
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(new byte [] {(byte)c}, GOOD_BYTES);
}
Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
filters.add(new PageRowFilter(MAX_PAGES));
filters.add(new RegExpRowFilter(".*regex.*", colvalues));
filters.add(new WhileMatchRowFilter(new StopRowFilter(Bytes.toBytes("yyy"))));
filters.add(new WhileMatchRowFilter(new RegExpRowFilter(".*match.*")));
filterMPALL = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ALL,
filters);
filterMPONE = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ONE,
filters);
}
/**
* Test "must pass one"
* @throws Exception
*/
public void testMPONE() throws Exception {
MPONETests(filterMPONE);
}
/**
* Test "must pass all"
* @throws Exception
*/
public void testMPALL() throws Exception {
MPALLTests(filterMPALL);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose filterMPALL to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
filterMPALL.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filterMPALL.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RowFilterSet();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
MPALLTests(newFilter);
}
private void MPONETests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause the WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", false);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", false);
// Accept several more rows such that PageRowFilter will exceed its limit.
for (int i=0; i<=MAX_PAGES-3; i++)
filter.rowProcessed(false, Bytes.toBytes("unimportant_key"));
// A row that should cause the RegExpRowFilter to filter this row, making
// all the filters return true and thus the RowFilterSet as well.
RFSAssertion(filter, "bad_column", true);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
}
private void MPALLTests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail.
boolean filtered = filter.filterColumn(Bytes.toBytes("regex_match"),
new byte [] { FIRST_CHAR }, BAD_BYTES);
assertTrue("Filtering on 'regex_match' and bad column data.", filtered);
filterMPALL.rowProcessed(filtered, Bytes.toBytes("regex_match"));
}
private void RFSAssertion(RowFilterInterface filter, String toTest,
boolean assertTrue) throws Exception {
byte [] testText = Bytes.toBytes(toTest);
boolean filtered = filter.filterRowKey(testText);
assertTrue("Filtering on '" + toTest + "'",
assertTrue? filtered : !filtered);
filter.rowProcessed(filtered, testText);
}
private void RFSAssertReset(RowFilterInterface filter) throws Exception{
assertTrue(filter.filterAllRemaining());
// Reset for continued testing
filter.reset();
assertFalse(filter.filterAllRemaining());
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;
import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests filter sets
*/
public class TestRowFilterSet extends TestCase {
RowFilterInterface filterMPALL;
RowFilterInterface filterMPONE;
static final int MAX_PAGES = 5;
final char FIRST_CHAR = 'a';
final char LAST_CHAR = 'e';
TreeMap<byte [], byte[]> colvalues;
static byte[] GOOD_BYTES = null;
static byte[] BAD_BYTES = null;
static {
try {
GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING);
BAD_BYTES = "def".getBytes(HConstants.UTF8_ENCODING);
} catch (UnsupportedEncodingException e) {
fail();
}
}
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
colvalues = new TreeMap<byte [], byte[]>(Bytes.BYTES_COMPARATOR);
for (char c = FIRST_CHAR; c < LAST_CHAR; c++) {
colvalues.put(new byte [] {(byte)c}, GOOD_BYTES);
}
Set<RowFilterInterface> filters = new HashSet<RowFilterInterface>();
filters.add(new PageRowFilter(MAX_PAGES));
filters.add(new RegExpRowFilter(".*regex.*", colvalues));
filters.add(new WhileMatchRowFilter(new StopRowFilter(Bytes.toBytes("yyy"))));
filters.add(new WhileMatchRowFilter(new RegExpRowFilter(".*match.*")));
filterMPALL = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ALL,
filters);
filterMPONE = new RowFilterSet(RowFilterSet.Operator.MUST_PASS_ONE,
filters);
}
/**
* Test "must pass one"
* @throws Exception
*/
public void testMPONE() throws Exception {
MPONETests(filterMPONE);
}
/**
* Test "must pass all"
* @throws Exception
*/
public void testMPALL() throws Exception {
MPALLTests(filterMPALL);
}
/**
* Test serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose filterMPALL to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
filterMPALL.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose filterMPALL.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new RowFilterSet();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
MPALLTests(newFilter);
}
private void MPONETests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause the WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", false);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", false);
// Accept several more rows such that PageRowFilter will exceed its limit.
for (int i=0; i<=MAX_PAGES-3; i++)
filter.rowProcessed(false, Bytes.toBytes("unimportant_key"));
// A row that should cause the RegExpRowFilter to filter this row, making
// all the filters return true and thus the RowFilterSet as well.
RFSAssertion(filter, "bad_column", true);
// Make sure the overall filterAllRemaining is unchanged (correct for
// MUST_PASS_ONE).
assertFalse(filter.filterAllRemaining());
}
private void MPALLTests(RowFilterInterface filter) throws Exception {
// A row that shouldn't cause any filters to return true.
RFSAssertion(filter, "regex_match", false);
// A row that should cause WhileMatchRowFilter to filter all remaining.
RFSAssertion(filter, "regex_only", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail and the
// StopRowFilter to filter all remaining.
RFSAssertion(filter, "yyy_match", true);
// Make sure the overall filterAllRemaining is changed (correct for
// MUST_PASS_ALL).
RFSAssertReset(filter);
// A row that should cause the RegExpRowFilter to fail.
boolean filtered = filter.filterColumn(Bytes.toBytes("regex_match"),
new byte [] { FIRST_CHAR }, BAD_BYTES);
assertTrue("Filtering on 'regex_match' and bad column data.", filtered);
filterMPALL.rowProcessed(filtered, Bytes.toBytes("regex_match"));
}
private void RFSAssertion(RowFilterInterface filter, String toTest,
boolean assertTrue) throws Exception {
byte [] testText = Bytes.toBytes(toTest);
boolean filtered = filter.filterRowKey(testText);
assertTrue("Filtering on '" + toTest + "'",
assertTrue? filtered : !filtered);
filter.rowProcessed(filtered, testText);
}
private void RFSAssertReset(RowFilterInterface filter) throws Exception{
assertTrue(filter.filterAllRemaining());
// Reset for continued testing
filter.reset();
assertFalse(filter.filterAllRemaining());
}
}

186
src/test/org/apache/hadoop/hbase/filter/TestStopRowFilter.java
View File

@ -1,93 +1,93 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests the stop row filter
*/
public class TestStopRowFilter extends TestCase {
private final byte [] STOP_ROW = Bytes.toBytes("stop_row");
private final byte [] GOOD_ROW = Bytes.toBytes("good_row");
private final byte [] PAST_STOP_ROW = Bytes.toBytes("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new StopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new StopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filterRowKey(GOOD_ROW));
assertTrue("Filtering on " + STOP_ROW, filter.filterRowKey(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterRowKey(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filterColumn(GOOD_ROW, null,
null));
assertTrue("Filtering on " + STOP_ROW, filter.filterColumn(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterColumn(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterRow(null));
assertFalse("Filter a null", filter.filterRowKey(null));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests the stop row filter
*/
public class TestStopRowFilter extends TestCase {
private final byte [] STOP_ROW = Bytes.toBytes("stop_row");
private final byte [] GOOD_ROW = Bytes.toBytes("good_row");
private final byte [] PAST_STOP_ROW = Bytes.toBytes("zzzzzz");
RowFilterInterface mainFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
mainFilter = new StopRowFilter(STOP_ROW);
}
/**
* Tests identification of the stop row
* @throws Exception
*/
public void testStopRowIdentification() throws Exception {
stopRowTests(mainFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
mainFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose mainFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
RowFilterInterface newFilter = new StopRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
stopRowTests(newFilter);
}
private void stopRowTests(RowFilterInterface filter) throws Exception {
assertFalse("Filtering on " + GOOD_ROW, filter.filterRowKey(GOOD_ROW));
assertTrue("Filtering on " + STOP_ROW, filter.filterRowKey(STOP_ROW));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterRowKey(PAST_STOP_ROW));
assertFalse("Filtering on " + GOOD_ROW, filter.filterColumn(GOOD_ROW, null,
null));
assertTrue("Filtering on " + STOP_ROW, filter.filterColumn(STOP_ROW, null, null));
assertTrue("Filtering on " + PAST_STOP_ROW, filter.filterColumn(PAST_STOP_ROW,
null, null));
assertFalse("FilterAllRemaining", filter.filterAllRemaining());
assertFalse("FilterNotNull", filter.filterRow(null));
assertFalse("Filter a null", filter.filterRowKey(null));
}
}

300
src/test/org/apache/hadoop/hbase/filter/TestWhileMatchRowFilter.java
View File

@ -1,150 +1,150 @@
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests for the while-match filter
*/
public class TestWhileMatchRowFilter extends TestCase {
WhileMatchRowFilter wmStopRowFilter;
WhileMatchRowFilter wmRegExpRowFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
wmStopRowFilter = new WhileMatchRowFilter(new StopRowFilter(
Bytes.toBytes("s")));
wmRegExpRowFilter = new WhileMatchRowFilter(new RegExpRowFilter(
".*regex.*"));
}
/**
* Tests while match stop row
* @throws Exception
*/
public void testWhileMatchStopRow() throws Exception {
whileMatchStopRowTests(wmStopRowFilter);
}
/**
* Tests while match regex
* @throws Exception
*/
public void testWhileMatchRegExp() throws Exception {
whileMatchRegExpTests(wmRegExpRowFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose wmRegExpRowFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
wmRegExpRowFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose wmRegExpRowFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
WhileMatchRowFilter newFilter = new WhileMatchRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
whileMatchRegExpTests(newFilter);
}
private void whileMatchStopRowTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "apples";
assertFalse("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// Test cases that should fail the row
toTest = "tuna";
assertTrue("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filterNotNull for functionality only (no switch-cases)
assertFalse("filter: filterNotNull", filter.filterRow(null));
}
private void whileMatchRegExpTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "regex_match";
assertFalse("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// Test cases that should fail the row
toTest = "not_a_match";
assertTrue("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filter(Text, Text, byte[]) for functionality only (no switch-cases)
toTest = "asdf_regex_hjkl";
assertFalse("filter: '" + toTest + "'", filter.filterColumn(Bytes.toBytes(toTest),
null, null));
}
}
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import org.apache.hadoop.hbase.util.Bytes;
import junit.framework.TestCase;
/**
* Tests for the while-match filter
*/
public class TestWhileMatchRowFilter extends TestCase {
WhileMatchRowFilter wmStopRowFilter;
WhileMatchRowFilter wmRegExpRowFilter;
/** {@inheritDoc} */
@Override
protected void setUp() throws Exception {
super.setUp();
wmStopRowFilter = new WhileMatchRowFilter(new StopRowFilter(
Bytes.toBytes("s")));
wmRegExpRowFilter = new WhileMatchRowFilter(new RegExpRowFilter(
".*regex.*"));
}
/**
* Tests while match stop row
* @throws Exception
*/
public void testWhileMatchStopRow() throws Exception {
whileMatchStopRowTests(wmStopRowFilter);
}
/**
* Tests while match regex
* @throws Exception
*/
public void testWhileMatchRegExp() throws Exception {
whileMatchRegExpTests(wmRegExpRowFilter);
}
/**
* Tests serialization
* @throws Exception
*/
public void testSerialization() throws Exception {
// Decompose wmRegExpRowFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
wmRegExpRowFilter.write(out);
out.close();
byte[] buffer = stream.toByteArray();
// Recompose wmRegExpRowFilter.
DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer));
WhileMatchRowFilter newFilter = new WhileMatchRowFilter();
newFilter.readFields(in);
// Ensure the serialization preserved the filter by running a full test.
whileMatchRegExpTests(newFilter);
}
private void whileMatchStopRowTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "apples";
assertFalse("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// Test cases that should fail the row
toTest = "tuna";
assertTrue("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filterNotNull for functionality only (no switch-cases)
assertFalse("filter: filterNotNull", filter.filterRow(null));
}
private void whileMatchRegExpTests(WhileMatchRowFilter filter) throws
Exception {
RowFilterInterface innerFilter = filter.getInternalFilter();
String toTest;
// Test cases that should pass the row
toTest = "regex_match";
assertFalse("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertFalse("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// Test cases that should fail the row
toTest = "not_a_match";
assertTrue("filter: '" + toTest + "'", filter.filterRowKey(Bytes.toBytes(toTest)));
assertTrue("innerFilter: '" + toTest + "'", innerFilter.filterRowKey(Bytes.toBytes(
toTest)));
// The difference in switch
assertTrue("filter: filterAllRemaining", filter.filterAllRemaining());
assertFalse("innerFilter: filterAllRemaining pre-reset",
innerFilter.filterAllRemaining());
// Test resetting
filter.reset();
assertFalse("filter: filterAllRemaining post-reset",
filter.filterAllRemaining());
// Test filter(Text, Text, byte[]) for functionality only (no switch-cases)
toTest = "asdf_regex_hjkl";
assertFalse("filter: '" + toTest + "'", filter.filterColumn(Bytes.toBytes(toTest),
null, null));
}
}