Apache
/
activemq
mirror of https://github.com/apache/activemq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Additional fix for: https://issues.apache.org/jira/browse/AMQ-3775 

On remove of inner ListNode entries the Iterator remove was not properly setting the Next page Id for the previous node leading to
the iterator not being able to traverse the entire list correctly.  

git-svn-id: https://svn.apache.org/repos/asf/activemq/trunk@1328739 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Timothy A. Bish 2012-04-21 21:57:31 +00:00
parent fc4f01cf5d
commit f77b2525a5
2 changed files with 200 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
kahadb/src/main/java/org/apache/kahadb/index/ListNode.java
View File

@ -31,10 +31,11 @@ import org.apache.kahadb.util.Marshaller;
import org.apache.kahadb.util.VariableMarshaller;
/**
* The ListNode class represents a node in the List object graph. It is stored in
* one overflowing Page of a PageFile.
* The ListNode class represents a node in the List object graph. It is stored
* in one overflowing Page of a PageFile.
*/
public final class ListNode<Key, Value> {
private final static boolean ADD_FIRST = true;
private final static boolean ADD_LAST = false;
@ -55,8 +56,8 @@ public final class ListNode<Key,Value> {
// The next page after this one.
private long next = ListIndex.NOT_SET;
static final class KeyValueEntry<Key, Value> extends LinkedNode<KeyValueEntry<Key, Value>> implements Entry<Key, Value>
{
static final class KeyValueEntry<Key, Value> extends LinkedNode<KeyValueEntry<Key, Value>> implements Entry<Key, Value> {
private final Key key;
private Value value;
@ -108,7 +109,8 @@ public final class ListNode<Key,Value> {
try {
nextEntry = index.loadNode(tx, current.next);
} catch (IOException unexpected) {
IllegalStateException e = new IllegalStateException("failed to load next: " + current.next + ", reason: " + unexpected.getLocalizedMessage());
IllegalStateException e = new IllegalStateException("failed to load next: " + current.next + ", reason: "
+ unexpected.getLocalizedMessage());
e.initCause(unexpected);
throw e;
}
@ -199,9 +201,8 @@ public final class ListNode<Key,Value> {
if (currentNode.isHead() && currentNode.isTail()) {
// store empty list
} else if (currentNode.isHead()) {
// merge next node into existing headNode
// as we don't want to change our headPageId b/c
// that is our identity
// merge next node into existing headNode as we don't want to
// change our headPageId b/c that is our identity
ListNode<Key, Value> headNode = currentNode;
nextEntry = getFromNextNode(); // will move currentNode
@ -220,6 +221,10 @@ public final class ListNode<Key,Value> {
previousNode.setNext(ListIndex.NOT_SET);
previousNode.store(tx);
targetList.setTailPageId(previousNode.getPageId());
} else {
toRemoveNode = currentNode;
previousNode.setNext(toRemoveNode.getNext());
previousNode.store(tx);
}
}
targetList.onRemove();
@ -258,7 +263,8 @@ public final class ListNode<Key,Value> {
public void writePayload(ListNode<Key, Value> node, DataOutput os) throws IOException {
os.writeLong(node.next);
short count = (short)node.entries.size(); // cast may truncate value...
short count = (short) node.entries.size(); // cast may truncate
// value...
if (count != node.entries.size()) {
throw new IOException("short over flow, too many entries in list: " + node.entries.size());
}
@ -278,9 +284,7 @@ public final class ListNode<Key,Value> {
node.setNext(is.readLong());
final short size = is.readShort();
for (short i = 0; i < size; i++) {
node.entries.addLast(
new KeyValueEntry(keyMarshaller.readPayload(is),
valueMarshaller.readPayload(is)));
node.entries.addLast(new KeyValueEntry(keyMarshaller.readPayload(is), valueMarshaller.readPayload(is)));
}
return node;
}
@ -318,9 +322,9 @@ public final class ListNode<Key,Value> {
private void store(Transaction tx, boolean addFirst) throws IOException {
try {
// When we split to a node of one element we can span multiple
// pages for that entry, otherwise we keep the entries on one
// page to avoid fragmented reads and segment the list traversal.
// When we split to a node of one element we can span multiple pages for that
// entry, otherwise we keep the entries on one page to avoid fragmented reads
// and segment the list traversal.
if (this.entries.size() == 1) {
getContainingList().storeNode(tx, this, true);
} else {
@ -477,5 +481,3 @@ public final class ListNode<Key,Value> {
return "[ListNode(" + (page != null ? page.getPageId() + "->" + next : "null") + ")[" + entries.size() + "]]";
}
}

146
kahadb/src/test/java/org/apache/kahadb/index/ListIndexTest.java
View File

@ -495,6 +495,152 @@ public class ListIndexTest extends IndexTestSupport {
}
}
public void testListLargeDataAddWithReverseRemove() throws Exception {
final int NUM_ITERATIONS = 100;
pf = new PageFile(directory, getClass().getName());
pf.setPageSize(4*1024);
pf.setEnablePageCaching(false);
pf.setWriteBatchSize(1);
pf.load();
tx = pf.tx();
long id = tx.allocate().getPageId();
ListIndex<String, SequenceSet> test = new ListIndex<String, SequenceSet>(pf, id);
test.setKeyMarshaller(StringMarshaller.INSTANCE);
test.setValueMarshaller(SequenceSet.Marshaller.INSTANCE);
test.load(tx);
tx.commit();
int expectedListEntries = 0;
int nextSequenceId = 0;
LOG.info("Loading up the ListIndex with "+NUM_ITERATIONS+" entires and sparsely populating the sequences.");
for (int i = 0; i < NUM_ITERATIONS; ++i) {
test.add(tx, String.valueOf(expectedListEntries++), new SequenceSet());
for (int j = 0; j < expectedListEntries; j++) {
SequenceSet sequenceSet = test.get(tx, String.valueOf(j));
int startSequenceId = nextSequenceId;
for (int ix = 0; ix < NUM_ITERATIONS; ix++) {
sequenceSet.add(nextSequenceId++);
test.put(tx, String.valueOf(j), sequenceSet);
}
sequenceSet = test.get(tx, String.valueOf(j));
for (int ix = 0; ix < NUM_ITERATIONS - 1; ix++) {
sequenceSet.remove(startSequenceId++);
test.put(tx, String.valueOf(j), sequenceSet);
}
}
}
LOG.info("Checking if Index has the expected number of entries.");
for (int i = 0; i < NUM_ITERATIONS; ++i) {
assertTrue("List should contain Key["+i+"]",test.containsKey(tx, String.valueOf(i)));
assertNotNull("List contents of Key["+i+"] should not be null", test.get(tx, String.valueOf(i)));
}
LOG.info("Index has the expected number of entries.");
assertEquals(expectedListEntries, test.size());
for (int i = NUM_ITERATIONS - 1; i >= 0; --i) {
LOG.debug("Size of ListIndex before removal of entry ["+i+"] is: " + test.size());
assertTrue("List should contain Key["+i+"]",test.containsKey(tx, String.valueOf(i)));
assertNotNull("List contents of Key["+i+"] should not be null", test.remove(tx, String.valueOf(i)));
LOG.debug("Size of ListIndex after removal of entry ["+i+"] is: " + test.size());
assertEquals(--expectedListEntries, test.size());
}
}
public void testListLargeDataAddAndNonSequentialRemove() throws Exception {
final int NUM_ITERATIONS = 100;
pf = new PageFile(directory, getClass().getName());
pf.setPageSize(4*1024);
pf.setEnablePageCaching(false);
pf.setWriteBatchSize(1);
pf.load();
tx = pf.tx();
long id = tx.allocate().getPageId();
ListIndex<String, SequenceSet> test = new ListIndex<String, SequenceSet>(pf, id);
test.setKeyMarshaller(StringMarshaller.INSTANCE);
test.setValueMarshaller(SequenceSet.Marshaller.INSTANCE);
test.load(tx);
tx.commit();
int expectedListEntries = 0;
int nextSequenceId = 0;
LOG.info("Loading up the ListIndex with "+NUM_ITERATIONS+" entires and sparsely populating the sequences.");
for (int i = 0; i < NUM_ITERATIONS; ++i) {
test.add(tx, String.valueOf(expectedListEntries++), new SequenceSet());
for (int j = 0; j < expectedListEntries; j++) {
SequenceSet sequenceSet = test.get(tx, String.valueOf(j));
int startSequenceId = nextSequenceId;
for (int ix = 0; ix < NUM_ITERATIONS; ix++) {
sequenceSet.add(nextSequenceId++);
test.put(tx, String.valueOf(j), sequenceSet);
}
sequenceSet = test.get(tx, String.valueOf(j));
for (int ix = 0; ix < NUM_ITERATIONS - 1; ix++) {
sequenceSet.remove(startSequenceId++);
test.put(tx, String.valueOf(j), sequenceSet);
}
}
}
LOG.info("Checking if Index has the expected number of entries.");
for (int i = 0; i < NUM_ITERATIONS; ++i) {
assertTrue("List should contain Key["+i+"]",test.containsKey(tx, String.valueOf(i)));
assertNotNull("List contents of Key["+i+"] should not be null", test.get(tx, String.valueOf(i)));
}
LOG.info("Index has the expected number of entries.");
assertEquals(expectedListEntries, test.size());
for (int i = 0; i < NUM_ITERATIONS; i += 2) {
LOG.debug("Size of ListIndex before removal of entry ["+i+"] is: " + test.size());
assertTrue("List should contain Key["+i+"]",test.containsKey(tx, String.valueOf(i)));
assertNotNull("List contents of Key["+i+"] should not be null", test.remove(tx, String.valueOf(i)));
LOG.debug("Size of ListIndex after removal of entry ["+i+"] is: " + test.size());
assertEquals(--expectedListEntries, test.size());
}
for (int i = NUM_ITERATIONS - 1; i > 0; i -= 2) {
LOG.debug("Size of ListIndex before removal of entry ["+i+"] is: " + test.size());
assertTrue("List should contain Key["+i+"]",test.containsKey(tx, String.valueOf(i)));
assertNotNull("List contents of Key["+i+"] should not be null", test.remove(tx, String.valueOf(i)));
LOG.debug("Size of ListIndex after removal of entry ["+i+"] is: " + test.size());
assertEquals(--expectedListEntries, test.size());
}
assertEquals(0, test.size());
}
static class HashSetStringMarshaller extends VariableMarshaller<HashSet<String>> {
final static HashSetStringMarshaller INSTANCE = new HashSetStringMarshaller();