HDDS-285. Create a generic Metadata Iterator. Contributed by Bharat Viswanadham.
This commit is contained in:
parent
5be9f4a5d0
commit
43db0cb518
|
@ -379,4 +379,9 @@ public class LevelDBStore implements MetadataStore {
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MetaStoreIterator<KeyValue> iterator() {
|
||||||
|
return new LevelDBStoreIterator(db.iterator());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,64 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.utils;
|
||||||
|
|
||||||
|
import org.iq80.leveldb.DBIterator;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.utils.MetadataStore.KeyValue;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* LevelDB store iterator.
|
||||||
|
*/
|
||||||
|
public class LevelDBStoreIterator implements MetaStoreIterator<KeyValue> {
|
||||||
|
|
||||||
|
|
||||||
|
private DBIterator levelDBIterator;
|
||||||
|
|
||||||
|
public LevelDBStoreIterator(DBIterator iterator) {
|
||||||
|
this.levelDBIterator = iterator;
|
||||||
|
levelDBIterator.seekToFirst();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return levelDBIterator.hasNext();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public KeyValue next() {
|
||||||
|
if(levelDBIterator.hasNext()) {
|
||||||
|
Map.Entry<byte[], byte[]> entry = levelDBIterator.next();
|
||||||
|
return KeyValue.create(entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
throw new NoSuchElementException("LevelDB Store has no more elements");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void seekToFirst() {
|
||||||
|
levelDBIterator.seekToFirst();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void seekToLast() {
|
||||||
|
levelDBIterator.seekToLast();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,39 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.utils;
|
||||||
|
|
||||||
|
import java.util.Iterator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Iterator for MetaDataStore DB.
|
||||||
|
* @param <T>
|
||||||
|
*/
|
||||||
|
interface MetaStoreIterator<T> extends Iterator<T> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* seek to first entry.
|
||||||
|
*/
|
||||||
|
void seekToFirst();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* seek to last entry.
|
||||||
|
*/
|
||||||
|
void seekToLast();
|
||||||
|
|
||||||
|
}
|
|
@ -169,4 +169,59 @@ public interface MetadataStore extends Closeable{
|
||||||
*/
|
*/
|
||||||
void iterate(byte[] from, EntryConsumer consumer)
|
void iterate(byte[] from, EntryConsumer consumer)
|
||||||
throws IOException;
|
throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the iterator for this metadata store.
|
||||||
|
* @return MetaStoreIterator
|
||||||
|
*/
|
||||||
|
MetaStoreIterator<KeyValue> iterator();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class used to represent the key and value pair of a db entry.
|
||||||
|
*/
|
||||||
|
class KeyValue {
|
||||||
|
|
||||||
|
private final byte[] key;
|
||||||
|
private final byte[] value;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* KeyValue Constructor, used to represent a key and value of a db entry.
|
||||||
|
* @param key
|
||||||
|
* @param value
|
||||||
|
*/
|
||||||
|
private KeyValue(byte[] key, byte[] value) {
|
||||||
|
this.key = key;
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return key.
|
||||||
|
* @return byte[]
|
||||||
|
*/
|
||||||
|
public byte[] getKey() {
|
||||||
|
byte[] result = new byte[key.length];
|
||||||
|
System.arraycopy(key, 0, result, 0, key.length);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return value.
|
||||||
|
* @return byte[]
|
||||||
|
*/
|
||||||
|
public byte[] getValue() {
|
||||||
|
byte[] result = new byte[value.length];
|
||||||
|
System.arraycopy(value, 0, result, 0, value.length);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a KeyValue pair.
|
||||||
|
* @param key
|
||||||
|
* @param value
|
||||||
|
* @return KeyValue object.
|
||||||
|
*/
|
||||||
|
public static KeyValue create(byte[] key, byte[] value) {
|
||||||
|
return new KeyValue(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -380,4 +380,9 @@ public class RocksDBStore implements MetadataStore {
|
||||||
return statMBeanName;
|
return statMBeanName;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MetaStoreIterator<KeyValue> iterator() {
|
||||||
|
return new RocksDBStoreIterator(db.newIterator());
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,66 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
package org.apache.hadoop.utils;
|
||||||
|
|
||||||
|
import org.rocksdb.RocksIterator;
|
||||||
|
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.utils.MetadataStore.KeyValue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* RocksDB store iterator.
|
||||||
|
*/
|
||||||
|
public class RocksDBStoreIterator implements MetaStoreIterator<KeyValue> {
|
||||||
|
|
||||||
|
private RocksIterator rocksDBIterator;
|
||||||
|
|
||||||
|
public RocksDBStoreIterator(RocksIterator iterator) {
|
||||||
|
this.rocksDBIterator = iterator;
|
||||||
|
rocksDBIterator.seekToFirst();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return rocksDBIterator.isValid();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public KeyValue next() {
|
||||||
|
if (rocksDBIterator.isValid()) {
|
||||||
|
KeyValue value = KeyValue.create(rocksDBIterator.key(), rocksDBIterator
|
||||||
|
.value());
|
||||||
|
rocksDBIterator.next();
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
throw new NoSuchElementException("RocksDB Store has no more elements");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void seekToFirst() {
|
||||||
|
rocksDBIterator.seekToFirst();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void seekToLast() {
|
||||||
|
rocksDBIterator.seekToLast();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -28,10 +28,10 @@ import org.apache.hadoop.hdfs.DFSUtil;
|
||||||
import org.apache.hadoop.hdfs.DFSUtilClient;
|
import org.apache.hadoop.hdfs.DFSUtilClient;
|
||||||
import org.apache.hadoop.ozone.OzoneConfigKeys;
|
import org.apache.hadoop.ozone.OzoneConfigKeys;
|
||||||
import org.apache.hadoop.test.GenericTestUtils;
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
import org.apache.hadoop.utils.MetadataStore.KeyValue;
|
||||||
import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
|
import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
|
||||||
import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
|
import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Assert;
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Rule;
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -48,10 +48,17 @@ import java.util.Collection;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
import static junit.framework.TestCase.assertTrue;
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
|
import static org.junit.Assert.assertNull;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
import static org.junit.runners.Parameterized.Parameters;
|
import static org.junit.runners.Parameterized.Parameters;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -109,6 +116,58 @@ public class TestMetadataStore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIterator() throws Exception {
|
||||||
|
Configuration conf = new OzoneConfiguration();
|
||||||
|
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
|
||||||
|
File dbDir = GenericTestUtils.getRandomizedTestDir();
|
||||||
|
MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
|
||||||
|
.setConf(conf)
|
||||||
|
.setCreateIfMissing(true)
|
||||||
|
.setDbFile(dbDir)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
//As database is empty, check whether iterator is working as expected or
|
||||||
|
// not.
|
||||||
|
MetaStoreIterator<KeyValue> metaStoreIterator = dbStore.iterator();
|
||||||
|
assertFalse(metaStoreIterator.hasNext());
|
||||||
|
try {
|
||||||
|
metaStoreIterator.next();
|
||||||
|
fail("testIterator failed");
|
||||||
|
} catch (NoSuchElementException ex) {
|
||||||
|
GenericTestUtils.assertExceptionContains("Store has no more elements",
|
||||||
|
ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
store.put(getBytes("a" + i), getBytes("a-value" + i));
|
||||||
|
}
|
||||||
|
|
||||||
|
metaStoreIterator = dbStore.iterator();
|
||||||
|
|
||||||
|
int i = 0;
|
||||||
|
while (metaStoreIterator.hasNext()) {
|
||||||
|
KeyValue val = metaStoreIterator.next();
|
||||||
|
assertEquals("a" + i, getString(val.getKey()));
|
||||||
|
assertEquals("a-value" + i, getString(val.getValue()));
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// As we have iterated all the keys in database, hasNext should return
|
||||||
|
// false and next() should throw NoSuchElement exception.
|
||||||
|
|
||||||
|
assertFalse(metaStoreIterator.hasNext());
|
||||||
|
try {
|
||||||
|
metaStoreIterator.next();
|
||||||
|
fail("testIterator failed");
|
||||||
|
} catch (NoSuchElementException ex) {
|
||||||
|
GenericTestUtils.assertExceptionContains("Store has no more elements",
|
||||||
|
ex);
|
||||||
|
}
|
||||||
|
FileUtils.deleteDirectory(dbDir);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testMetaStoreConfigDifferentFromType() throws IOException {
|
public void testMetaStoreConfigDifferentFromType() throws IOException {
|
||||||
|
|
||||||
|
@ -183,17 +242,17 @@ public class TestMetadataStore {
|
||||||
public void testGetDelete() throws IOException {
|
public void testGetDelete() throws IOException {
|
||||||
for (int i=0; i<10; i++) {
|
for (int i=0; i<10; i++) {
|
||||||
byte[] va = store.get(getBytes("a" + i));
|
byte[] va = store.get(getBytes("a" + i));
|
||||||
Assert.assertEquals("a-value" + i, getString(va));
|
assertEquals("a-value" + i, getString(va));
|
||||||
|
|
||||||
byte[] vb = store.get(getBytes("b" + i));
|
byte[] vb = store.get(getBytes("b" + i));
|
||||||
Assert.assertEquals("b-value" + i, getString(vb));
|
assertEquals("b-value" + i, getString(vb));
|
||||||
}
|
}
|
||||||
|
|
||||||
String keyToDel = "del-" + UUID.randomUUID().toString();
|
String keyToDel = "del-" + UUID.randomUUID().toString();
|
||||||
store.put(getBytes(keyToDel), getBytes(keyToDel));
|
store.put(getBytes(keyToDel), getBytes(keyToDel));
|
||||||
Assert.assertEquals(keyToDel, getString(store.get(getBytes(keyToDel))));
|
assertEquals(keyToDel, getString(store.get(getBytes(keyToDel))));
|
||||||
store.delete(getBytes(keyToDel));
|
store.delete(getBytes(keyToDel));
|
||||||
Assert.assertEquals(null, store.get(getBytes(keyToDel)));
|
assertEquals(null, store.get(getBytes(keyToDel)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -228,8 +287,8 @@ public class TestMetadataStore {
|
||||||
k = getString(current.getKey());
|
k = getString(current.getKey());
|
||||||
v = getString(current.getValue());
|
v = getString(current.getValue());
|
||||||
}
|
}
|
||||||
Assert.assertEquals(peekKey, k);
|
assertEquals(peekKey, k);
|
||||||
Assert.assertEquals(v, getExpectedValue(peekKey));
|
assertEquals(v, getExpectedValue(peekKey));
|
||||||
|
|
||||||
// Look for prev
|
// Look for prev
|
||||||
k = null;
|
k = null;
|
||||||
|
@ -240,8 +299,8 @@ public class TestMetadataStore {
|
||||||
k = getString(prev.getKey());
|
k = getString(prev.getKey());
|
||||||
v = getString(prev.getValue());
|
v = getString(prev.getValue());
|
||||||
}
|
}
|
||||||
Assert.assertEquals(prevKey, k);
|
assertEquals(prevKey, k);
|
||||||
Assert.assertEquals(v, getExpectedValue(prevKey));
|
assertEquals(v, getExpectedValue(prevKey));
|
||||||
|
|
||||||
// Look for next
|
// Look for next
|
||||||
k = null;
|
k = null;
|
||||||
|
@ -252,8 +311,8 @@ public class TestMetadataStore {
|
||||||
k = getString(next.getKey());
|
k = getString(next.getKey());
|
||||||
v = getString(next.getValue());
|
v = getString(next.getValue());
|
||||||
}
|
}
|
||||||
Assert.assertEquals(nextKey, k);
|
assertEquals(nextKey, k);
|
||||||
Assert.assertEquals(v, getExpectedValue(nextKey));
|
assertEquals(v, getExpectedValue(nextKey));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -271,9 +330,9 @@ public class TestMetadataStore {
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
|
||||||
Assert.assertFalse(result.isEmpty());
|
assertFalse(result.isEmpty());
|
||||||
for (int i=0; i<result.size(); i++) {
|
for (int i=0; i<result.size(); i++) {
|
||||||
Assert.assertEquals("b-value" + (i+1), result.get(i));
|
assertEquals("b-value" + (i+1), result.get(i));
|
||||||
}
|
}
|
||||||
|
|
||||||
// iterate from a non exist key
|
// iterate from a non exist key
|
||||||
|
@ -282,7 +341,7 @@ public class TestMetadataStore {
|
||||||
result.add(getString(v));
|
result.add(getString(v));
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
Assert.assertTrue(result.isEmpty());
|
assertTrue(result.isEmpty());
|
||||||
|
|
||||||
// iterate from the beginning
|
// iterate from the beginning
|
||||||
result.clear();
|
result.clear();
|
||||||
|
@ -290,7 +349,7 @@ public class TestMetadataStore {
|
||||||
result.add(getString(v));
|
result.add(getString(v));
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
Assert.assertEquals(20, result.size());
|
assertEquals(20, result.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -299,66 +358,66 @@ public class TestMetadataStore {
|
||||||
|
|
||||||
// Set empty startKey will return values from beginning.
|
// Set empty startKey will return values from beginning.
|
||||||
result = store.getRangeKVs(null, 5);
|
result = store.getRangeKVs(null, 5);
|
||||||
Assert.assertEquals(5, result.size());
|
assertEquals(5, result.size());
|
||||||
Assert.assertEquals("a-value2", getString(result.get(2).getValue()));
|
assertEquals("a-value2", getString(result.get(2).getValue()));
|
||||||
|
|
||||||
// Empty list if startKey doesn't exist.
|
// Empty list if startKey doesn't exist.
|
||||||
result = store.getRangeKVs(getBytes("a12"), 5);
|
result = store.getRangeKVs(getBytes("a12"), 5);
|
||||||
Assert.assertEquals(0, result.size());
|
assertEquals(0, result.size());
|
||||||
|
|
||||||
// Returns max available entries after a valid startKey.
|
// Returns max available entries after a valid startKey.
|
||||||
result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH);
|
result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH);
|
||||||
Assert.assertEquals(10, result.size());
|
assertEquals(10, result.size());
|
||||||
Assert.assertEquals("b0", getString(result.get(0).getKey()));
|
assertEquals("b0", getString(result.get(0).getKey()));
|
||||||
Assert.assertEquals("b-value0", getString(result.get(0).getValue()));
|
assertEquals("b-value0", getString(result.get(0).getValue()));
|
||||||
result = store.getRangeKVs(getBytes("b0"), 5);
|
result = store.getRangeKVs(getBytes("b0"), 5);
|
||||||
Assert.assertEquals(5, result.size());
|
assertEquals(5, result.size());
|
||||||
|
|
||||||
// Both startKey and count are honored.
|
// Both startKey and count are honored.
|
||||||
result = store.getRangeKVs(getBytes("a9"), 2);
|
result = store.getRangeKVs(getBytes("a9"), 2);
|
||||||
Assert.assertEquals(2, result.size());
|
assertEquals(2, result.size());
|
||||||
Assert.assertEquals("a9", getString(result.get(0).getKey()));
|
assertEquals("a9", getString(result.get(0).getKey()));
|
||||||
Assert.assertEquals("a-value9", getString(result.get(0).getValue()));
|
assertEquals("a-value9", getString(result.get(0).getValue()));
|
||||||
Assert.assertEquals("b0", getString(result.get(1).getKey()));
|
assertEquals("b0", getString(result.get(1).getKey()));
|
||||||
Assert.assertEquals("b-value0", getString(result.get(1).getValue()));
|
assertEquals("b-value0", getString(result.get(1).getValue()));
|
||||||
|
|
||||||
// Filter keys by prefix.
|
// Filter keys by prefix.
|
||||||
// It should returns all "b*" entries.
|
// It should returns all "b*" entries.
|
||||||
MetadataKeyFilter filter1 = new KeyPrefixFilter().addFilter("b");
|
MetadataKeyFilter filter1 = new KeyPrefixFilter().addFilter("b");
|
||||||
result = store.getRangeKVs(null, 100, filter1);
|
result = store.getRangeKVs(null, 100, filter1);
|
||||||
Assert.assertEquals(10, result.size());
|
assertEquals(10, result.size());
|
||||||
Assert.assertTrue(result.stream().allMatch(entry ->
|
assertTrue(result.stream().allMatch(entry ->
|
||||||
new String(entry.getKey()).startsWith("b")
|
new String(entry.getKey()).startsWith("b")
|
||||||
));
|
));
|
||||||
Assert.assertEquals(20, filter1.getKeysScannedNum());
|
assertEquals(20, filter1.getKeysScannedNum());
|
||||||
Assert.assertEquals(10, filter1.getKeysHintedNum());
|
assertEquals(10, filter1.getKeysHintedNum());
|
||||||
result = store.getRangeKVs(null, 3, filter1);
|
result = store.getRangeKVs(null, 3, filter1);
|
||||||
Assert.assertEquals(3, result.size());
|
assertEquals(3, result.size());
|
||||||
result = store.getRangeKVs(getBytes("b3"), 1, filter1);
|
result = store.getRangeKVs(getBytes("b3"), 1, filter1);
|
||||||
Assert.assertEquals("b-value3", getString(result.get(0).getValue()));
|
assertEquals("b-value3", getString(result.get(0).getValue()));
|
||||||
|
|
||||||
// Define a customized filter that filters keys by suffix.
|
// Define a customized filter that filters keys by suffix.
|
||||||
// Returns all "*2" entries.
|
// Returns all "*2" entries.
|
||||||
MetadataKeyFilter filter2 = (preKey, currentKey, nextKey)
|
MetadataKeyFilter filter2 = (preKey, currentKey, nextKey)
|
||||||
-> getString(currentKey).endsWith("2");
|
-> getString(currentKey).endsWith("2");
|
||||||
result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2);
|
result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2);
|
||||||
Assert.assertEquals(2, result.size());
|
assertEquals(2, result.size());
|
||||||
Assert.assertEquals("a2", getString(result.get(0).getKey()));
|
assertEquals("a2", getString(result.get(0).getKey()));
|
||||||
Assert.assertEquals("b2", getString(result.get(1).getKey()));
|
assertEquals("b2", getString(result.get(1).getKey()));
|
||||||
result = store.getRangeKVs(null, 1, filter2);
|
result = store.getRangeKVs(null, 1, filter2);
|
||||||
Assert.assertEquals(1, result.size());
|
assertEquals(1, result.size());
|
||||||
Assert.assertEquals("a2", getString(result.get(0).getKey()));
|
assertEquals("a2", getString(result.get(0).getKey()));
|
||||||
|
|
||||||
// Apply multiple filters.
|
// Apply multiple filters.
|
||||||
result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2);
|
result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2);
|
||||||
Assert.assertEquals(1, result.size());
|
assertEquals(1, result.size());
|
||||||
Assert.assertEquals("b2", getString(result.get(0).getKey()));
|
assertEquals("b2", getString(result.get(0).getKey()));
|
||||||
Assert.assertEquals("b-value2", getString(result.get(0).getValue()));
|
assertEquals("b-value2", getString(result.get(0).getValue()));
|
||||||
|
|
||||||
// If filter is null, no effect.
|
// If filter is null, no effect.
|
||||||
result = store.getRangeKVs(null, 1, null);
|
result = store.getRangeKVs(null, 1, null);
|
||||||
Assert.assertEquals(1, result.size());
|
assertEquals(1, result.size());
|
||||||
Assert.assertEquals("a0", getString(result.get(0).getKey()));
|
assertEquals("a0", getString(result.get(0).getKey()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -368,16 +427,16 @@ public class TestMetadataStore {
|
||||||
// Suppose to return a2 and b2
|
// Suppose to return a2 and b2
|
||||||
List<Map.Entry<byte[], byte[]>> result =
|
List<Map.Entry<byte[], byte[]>> result =
|
||||||
store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter);
|
store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter);
|
||||||
Assert.assertEquals(2, result.size());
|
assertEquals(2, result.size());
|
||||||
Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
|
assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
|
||||||
Assert.assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey()));
|
assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey()));
|
||||||
|
|
||||||
// Suppose to return just a2, because when it iterates to a3,
|
// Suppose to return just a2, because when it iterates to a3,
|
||||||
// the filter no long matches and it should stop from there.
|
// the filter no long matches and it should stop from there.
|
||||||
result = store.getSequentialRangeKVs(null,
|
result = store.getSequentialRangeKVs(null,
|
||||||
MAX_GETRANGE_LENGTH, suffixFilter);
|
MAX_GETRANGE_LENGTH, suffixFilter);
|
||||||
Assert.assertEquals(1, result.size());
|
assertEquals(1, result.size());
|
||||||
Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
|
assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -385,10 +444,10 @@ public class TestMetadataStore {
|
||||||
List<Map.Entry<byte[], byte[]>> result = null;
|
List<Map.Entry<byte[], byte[]>> result = null;
|
||||||
|
|
||||||
result = store.getRangeKVs(null, 0);
|
result = store.getRangeKVs(null, 0);
|
||||||
Assert.assertEquals(0, result.size());
|
assertEquals(0, result.size());
|
||||||
|
|
||||||
result = store.getRangeKVs(null, 1);
|
result = store.getRangeKVs(null, 1);
|
||||||
Assert.assertEquals(1, result.size());
|
assertEquals(1, result.size());
|
||||||
|
|
||||||
// Count less than zero is invalid.
|
// Count less than zero is invalid.
|
||||||
expectedException.expect(IllegalArgumentException.class);
|
expectedException.expect(IllegalArgumentException.class);
|
||||||
|
@ -401,7 +460,7 @@ public class TestMetadataStore {
|
||||||
// If startKey is invalid, the returned list should be empty.
|
// If startKey is invalid, the returned list should be empty.
|
||||||
List<Map.Entry<byte[], byte[]>> kvs =
|
List<Map.Entry<byte[], byte[]>> kvs =
|
||||||
store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH);
|
store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH);
|
||||||
Assert.assertEquals(kvs.size(), 0);
|
assertEquals(kvs.size(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -421,13 +480,13 @@ public class TestMetadataStore {
|
||||||
dbStore.put(getBytes("key1"), getBytes("value1"));
|
dbStore.put(getBytes("key1"), getBytes("value1"));
|
||||||
dbStore.put(getBytes("key2"), getBytes("value2"));
|
dbStore.put(getBytes("key2"), getBytes("value2"));
|
||||||
|
|
||||||
Assert.assertFalse(dbStore.isEmpty());
|
assertFalse(dbStore.isEmpty());
|
||||||
Assert.assertTrue(dbDir.exists());
|
assertTrue(dbDir.exists());
|
||||||
Assert.assertTrue(dbDir.listFiles().length > 0);
|
assertTrue(dbDir.listFiles().length > 0);
|
||||||
|
|
||||||
dbStore.destroy();
|
dbStore.destroy();
|
||||||
|
|
||||||
Assert.assertFalse(dbDir.exists());
|
assertFalse(dbDir.exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -469,7 +528,7 @@ public class TestMetadataStore {
|
||||||
return it.hasNext() && it.next().equals(getString(key));
|
return it.hasNext() && it.next().equals(getString(key));
|
||||||
});
|
});
|
||||||
|
|
||||||
Assert.assertEquals(8, count.get());
|
assertEquals(8, count.get());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -482,52 +541,51 @@ public class TestMetadataStore {
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
exception = e;
|
exception = e;
|
||||||
}
|
}
|
||||||
Assert.assertTrue(
|
assertTrue(exception.getMessage().contains("KeyPrefix: b already " +
|
||||||
exception.getMessage().contains("KeyPrefix: b already rejected"));
|
"rejected"));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
new KeyPrefixFilter().addFilter("b0").addFilter("b", true);
|
new KeyPrefixFilter().addFilter("b0").addFilter("b", true);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
exception = e;
|
exception = e;
|
||||||
}
|
}
|
||||||
Assert.assertTrue(
|
assertTrue(exception.getMessage().contains("KeyPrefix: b already " +
|
||||||
exception.getMessage().contains("KeyPrefix: b already accepted"));
|
"accepted"));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
new KeyPrefixFilter().addFilter("b", true).addFilter("b0");
|
new KeyPrefixFilter().addFilter("b", true).addFilter("b0");
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
exception = e;
|
exception = e;
|
||||||
}
|
}
|
||||||
Assert.assertTrue(
|
assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " +
|
||||||
exception.getMessage().contains("KeyPrefix: b0 already rejected"));
|
"rejected"));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
new KeyPrefixFilter().addFilter("b").addFilter("b0", true);
|
new KeyPrefixFilter().addFilter("b").addFilter("b0", true);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
exception = e;
|
exception = e;
|
||||||
}
|
}
|
||||||
Assert.assertTrue(
|
assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " +
|
||||||
exception.getMessage().contains("KeyPrefix: b0 already accepted"));
|
"accepted"));
|
||||||
|
|
||||||
MetadataKeyFilter filter1 = new KeyPrefixFilter(true)
|
MetadataKeyFilter filter1 = new KeyPrefixFilter(true)
|
||||||
.addFilter("a0")
|
.addFilter("a0")
|
||||||
.addFilter("a1")
|
.addFilter("a1")
|
||||||
.addFilter("b", true);
|
.addFilter("b", true);
|
||||||
result = store.getRangeKVs(null, 100, filter1);
|
result = store.getRangeKVs(null, 100, filter1);
|
||||||
Assert.assertEquals(2, result.size());
|
assertEquals(2, result.size());
|
||||||
Assert.assertTrue(result.stream()
|
assertTrue(result.stream().anyMatch(entry -> new String(entry.getKey())
|
||||||
.anyMatch(entry -> new String(entry.getKey()).startsWith("a0"))
|
.startsWith("a0")) && result.stream().anyMatch(entry -> new String(
|
||||||
&& result.stream()
|
entry.getKey()).startsWith("a1")));
|
||||||
.anyMatch(entry -> new String(entry.getKey()).startsWith("a1")));
|
|
||||||
|
|
||||||
filter1 = new KeyPrefixFilter(true).addFilter("b", true);
|
filter1 = new KeyPrefixFilter(true).addFilter("b", true);
|
||||||
result = store.getRangeKVs(null, 100, filter1);
|
result = store.getRangeKVs(null, 100, filter1);
|
||||||
Assert.assertEquals(0, result.size());
|
assertEquals(0, result.size());
|
||||||
|
|
||||||
filter1 = new KeyPrefixFilter().addFilter("b", true);
|
filter1 = new KeyPrefixFilter().addFilter("b", true);
|
||||||
result = store.getRangeKVs(null, 100, filter1);
|
result = store.getRangeKVs(null, 100, filter1);
|
||||||
Assert.assertEquals(10, result.size());
|
assertEquals(10, result.size());
|
||||||
Assert.assertTrue(result.stream()
|
assertTrue(result.stream().allMatch(entry -> new String(entry.getKey())
|
||||||
.allMatch(entry -> new String(entry.getKey()).startsWith("a")));
|
.startsWith("a")));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue