From e11ece117cefd3a50ad43920de2a3246e3305266 Mon Sep 17 00:00:00 2001 From: Gary Tully Date: Wed, 25 May 2011 14:28:44 +0000 Subject: [PATCH] add list index that will keep x entries per page, basis for plist that is more space efficient git-svn-id: https://svn.apache.org/repos/asf/activemq/trunk@1127543 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/kahadb/index/ListIndex.java | 257 +++++++++++ .../org/apache/kahadb/index/ListNode.java | 430 ++++++++++++++++++ .../java/org/apache/kahadb/page/PageFile.java | 11 +- .../org/apache/kahadb/util/LinkedNode.java | 4 - .../apache/kahadb/util/LinkedNodeList.java | 2 +- .../apache/kahadb/index/ListIndexTest.java | 326 +++++++++++++ 6 files changed, 1021 insertions(+), 9 deletions(-) create mode 100644 kahadb/src/main/java/org/apache/kahadb/index/ListIndex.java create mode 100644 kahadb/src/main/java/org/apache/kahadb/index/ListNode.java create mode 100644 kahadb/src/test/java/org/apache/kahadb/index/ListIndexTest.java diff --git a/kahadb/src/main/java/org/apache/kahadb/index/ListIndex.java b/kahadb/src/main/java/org/apache/kahadb/index/ListIndex.java new file mode 100644 index 0000000000..f1d7cff30e --- /dev/null +++ b/kahadb/src/main/java/org/apache/kahadb/index/ListIndex.java @@ -0,0 +1,257 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kahadb.index; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.kahadb.page.Page; +import org.apache.kahadb.page.PageFile; +import org.apache.kahadb.page.Transaction; +import org.apache.kahadb.util.Marshaller; + +public class ListIndex implements Index { + + private static final Logger LOG = LoggerFactory.getLogger(ListIndex.class); + + protected PageFile pageFile; + protected long headPageId; + protected long tailPageId; + private long size; + + protected AtomicBoolean loaded = new AtomicBoolean(); + + private final ListNode.Marshaller marshaller = new ListNode.Marshaller(this); + private Marshaller keyMarshaller; + private Marshaller valueMarshaller; + + public ListIndex(PageFile pageFile, long rootPageId) { + this.pageFile = pageFile; + this.headPageId = rootPageId; + } + + synchronized public void load(Transaction tx) throws IOException { + if (loaded.compareAndSet(false, true)) { + LOG.debug("loading"); + if( keyMarshaller == null ) { + throw new IllegalArgumentException("The key marshaller must be set before loading the ListIndex"); + } + if( valueMarshaller == null ) { + throw new IllegalArgumentException("The value marshaller must be set before loading the ListIndex"); + } + + final Page> p = tx.load(headPageId, null); + if( p.getType() == Page.PAGE_FREE_TYPE ) { + // Need to initialize it.. + ListNode root = createNode(p, null); + storeNode(tx, root, true); + tailPageId = headPageId; + } else { + ListNode node = loadNode(tx, headPageId, null); + size += node.size(tx); + while (node.getNext() != -1) { + node = loadNode(tx, node.getNext(), node); + size += node.size(tx); + tailPageId = node.getPageId(); + } + } + } + } + + synchronized public void unload(Transaction tx) { + if (loaded.compareAndSet(true, false)) { + } + } + + protected ListNode getHead(Transaction tx) throws IOException { + return loadNode(tx, headPageId, null); + } + + protected ListNode getTail(Transaction tx) throws IOException { + return loadNode(tx, tailPageId, null); + } + + synchronized public boolean containsKey(Transaction tx, Key key) throws IOException { + assertLoaded(); + for (Iterator> iterator = iterator(tx); iterator.hasNext(); ) { + Map.Entry candidate = iterator.next(); + if (key.equals(candidate.getKey())) { + return true; + } + } + return false; + } + + synchronized public Value get(Transaction tx, Key key) throws IOException { + assertLoaded(); + for (Iterator> iterator = iterator(tx); iterator.hasNext(); ) { + Map.Entry candidate = iterator.next(); + if (key.equals(candidate.getKey())) { + return candidate.getValue(); + } + } + return null; + } + + /** + * appends to the list + * @return null + */ + synchronized public Value put(Transaction tx, Key key, Value value) throws IOException { + return add(tx, key, value); + } + + synchronized public Value add(Transaction tx, Key key, Value value) throws IOException { + assertLoaded(); + getTail(tx).put(tx, key, value); + size ++; + return null; + } + + synchronized public Value addFirst(Transaction tx, Key key, Value value) throws IOException { + assertLoaded(); + getHead(tx).addFirst(tx, key, value); + size++; + return null; + } + + synchronized public Value remove(Transaction tx, Key key) throws IOException { + assertLoaded(); + for (Iterator> iterator = iterator(tx); iterator.hasNext(); ) { + Map.Entry candidate = iterator.next(); + if (key.equals(candidate.getKey())) { + iterator.remove(); + return candidate.getValue(); + } + } + return null; + } + + public void onRemove() { + size--; + } + + public boolean isTransient() { + return false; + } + + synchronized public void clear(Transaction tx) throws IOException { + for (Iterator> iterator = listNodeIterator(tx); iterator.hasNext(); ) { + ListNodecandidate = iterator.next(); + candidate.clear(tx); + } + size = 0; + } + + synchronized public Iterator> listNodeIterator(Transaction tx) throws IOException { + return getHead(tx).listNodeIterator(tx); + } + + synchronized public boolean isEmpty(final Transaction tx) throws IOException { + return getHead(tx).isEmpty(tx); + } + + synchronized public Iterator> iterator(final Transaction tx) throws IOException { + return getHead(tx).iterator(tx); + } + + synchronized public Iterator> iterator(final Transaction tx, int initialPosition) throws IOException { + return getHead(tx).iterator(tx, initialPosition); + } + + synchronized public Map.Entry getFirst(Transaction tx) throws IOException { + return getHead(tx).getFirst(tx); + } + + synchronized public Map.Entry getLast(Transaction tx) throws IOException { + return getTail(tx).getLast(tx); + } + + private void assertLoaded() throws IllegalStateException { + if( !loaded.get() ) { + throw new IllegalStateException("TheListIndex is not loaded"); + } + } + + ListNode loadNode(Transaction tx, long pageId, ListNode parent) throws IOException { + Page> page = tx.load(pageId, marshaller); + ListNode node = page.get(); + node.setPage(page); + node.setParent(parent); + return node; + } + + ListNode createNode(Page> p, ListNode parent) throws IOException { + ListNode node = new ListNode(this); + node.setPage(p); + node.setParent(parent); + node.setEmpty(); + p.set(node); + return node; + } + + ListNode createNode(Transaction tx, ListNode parent) throws IOException { + Page> page = tx.load(tx.allocate(1).getPageId(), marshaller); + ListNode node = new ListNode(this); + node.setPage(page); + node.setParent(parent); + node.setEmpty(); + page.set(node); + return node; + } + + void storeNode(Transaction tx, ListNode node, boolean overflow) throws IOException { + tx.store(node.getPage(), marshaller, overflow); + } + + public PageFile getPageFile() { + return pageFile; + } + public long getHeadPageId() { + return headPageId; + } + + public void setHeadPageId(long headPageId) { + this.headPageId = headPageId; + } + + public Marshaller getKeyMarshaller() { + return keyMarshaller; + } + public void setKeyMarshaller(Marshaller keyMarshaller) { + this.keyMarshaller = keyMarshaller; + } + + public Marshaller getValueMarshaller() { + return valueMarshaller; + } + public void setValueMarshaller(Marshaller valueMarshaller) { + this.valueMarshaller = valueMarshaller; + } + + public void setTailPageId(long tailPageId) { + this.tailPageId = tailPageId; + } + + public long size() { + return size; + } +} diff --git a/kahadb/src/main/java/org/apache/kahadb/index/ListNode.java b/kahadb/src/main/java/org/apache/kahadb/index/ListNode.java new file mode 100644 index 0000000000..a5a6cc8e59 --- /dev/null +++ b/kahadb/src/main/java/org/apache/kahadb/index/ListNode.java @@ -0,0 +1,430 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kahadb.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import javax.swing.plaf.basic.BasicInternalFrameTitlePane; +import org.apache.kahadb.index.BTreeIndex.Prefixer; +import org.apache.kahadb.page.Page; +import org.apache.kahadb.page.Transaction; +import org.apache.kahadb.util.LinkedNode; +import org.apache.kahadb.util.LinkedNodeList; +import org.apache.kahadb.util.VariableMarshaller; +import sun.plugin.dom.exception.InvalidStateException; +import sun.tools.tree.ReturnStatement; +import sun.util.resources.CurrencyNames_th_TH; + +/** + * The ListNode class represents a node in the List object graph. It is stored in + * one overflowing Page of a PageFile. + */ +public final class ListNode { + + // The index that this node is part of. + private final ListIndex index; + // The parent node or null if this is the root node of the List + private ListNode parent; + // The page associated with this node + private Page> page; + + protected LinkedNodeList> entries = new LinkedNodeList>(); + + // The next page after this one. + private long next = -1; + + public int size(Transaction tx) { + return entries.size(); + } + + static final class KeyValueEntry extends LinkedNode> implements Entry + { + private final Key key; + private final Value value; + + public KeyValueEntry(Key key, Value value) { + this.key = key; + this.value = value; + } + + public Key getKey() { + return key; + } + + public Value getValue() { + return value; + } + + public Value setValue(Value value) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "{" + key + ":" + value + "}"; + } + } + + private final class ListNodeIterator implements Iterator> { + + private final Transaction tx; + ListNode nextEntry; + + private ListNodeIterator(Transaction tx, ListNode current) throws IOException { + this.tx = tx; + nextEntry = current; + } + + public boolean hasNext() { + return nextEntry !=null; + } + + public ListNode next() { + ListNode current = nextEntry; + if( nextEntry !=null ) { + if (nextEntry.next != -1) { + try { + nextEntry = index.loadNode(tx, current.next, current); + } catch (IOException unexpected) { + InvalidStateException e = new InvalidStateException("failed to load next: " + current.next + ", reason: " + unexpected.getLocalizedMessage()); + e.initCause(unexpected); + throw e; + } + } else { + nextEntry = null; + } + } + return current; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + private final class ListIterator implements Iterator> { + + private final Transaction tx; + ListNode current; + KeyValueEntry nextEntry; + KeyValueEntry toRemove; + + private ListIterator(Transaction tx, ListNode current, int nextIndex) throws IOException { + this.tx = tx; + this.current = current; + nextEntry = current.entries.getHead(); + if (nextIndex > 0) { + for (int i=0; i next() { + if( nextEntry !=null ) { + toRemove = nextEntry; + nextEntry=toRemove.getNext(); + return toRemove; + } else { + throw new NoSuchElementException(); + } + } + + public void remove() { + if (toRemove == null) { + throw new InvalidStateException("can only remove once, call next again"); + } + try { + doRemove(tx, current, toRemove); + index.onRemove(); + toRemove = null; + } catch (IOException unexpected) { + InvalidStateException e = new InvalidStateException(unexpected.getLocalizedMessage()); + e.initCause(unexpected); + throw e; + } + } + } + + /** + * The Marshaller is used to store and load the data in the ListNode into a Page. + * + * @param + * @param + */ + static public class Marshaller extends VariableMarshaller> { + private final ListIndex index; + + public Marshaller(ListIndex index) { + this.index = index; + } + + public void writePayload(ListNode node, DataOutput os) throws IOException { + // Write the keys + short count = (short)node.entries.size(); // cast may truncate value... + if( count != node.entries.size() ) { + throw new IOException("short over flow, too many entries in list: " + node.entries.size()); + } + + os.writeShort(count); + KeyValueEntry entry = node.entries.getHead(); + while (entry != null) { + index.getKeyMarshaller().writePayload((Key) entry.getKey(), os); + index.getValueMarshaller().writePayload((Value) entry.getValue(), os); + entry = entry.getNext(); + } + } + + @SuppressWarnings("unchecked") + public ListNode readPayload(DataInput is) throws IOException { + ListNode node = new ListNode(index); + final short size = is.readShort(); + for (short i = 0; i < size; i++) { + node.entries.addLast( + new KeyValueEntry(index.getKeyMarshaller().readPayload(is), + index.getValueMarshaller().readPayload(is))); + } + return node; + } + } + + public ListNode(ListIndex index) { + this.index = index; + } + + public void setEmpty() { + } + + public Value remove(Transaction tx, Key key) throws IOException { + Value result = null; + KeyValueEntry entry = entries.getHead(); + while (entry != null) { + if (entry.getKey().equals(key)) { + result = entry.getValue(); + doRemove(tx, this, entry); + break; + } + entry = entry.getNext(); + } + return result; + } + + private void doRemove(Transaction tx, ListNode current, KeyValueEntry entry) throws IOException { + entry.unlink(); + if (current.entries.isEmpty()) { + if (current.getPageId() == index.getHeadPageId()) { + if (current.getNext() != -1) { + // new head + index.setHeadPageId(current.getNext()); + tx.free(current.getPageId()); + } + } else { + // need to unlink the node + current.parent.setNext(current.getNext()); + tx.free(current.getPageId()); + index.storeNode(tx, current.parent, false); + } + } else { + store(tx, true); + } + } + + public Value put(Transaction tx, Key key, Value value) throws IOException { + if (key == null) { + throw new IllegalArgumentException("Key cannot be null"); + } + entries.addLast(new KeyValueEntry(key, value)); + store(tx, false); + return null; + } + + public Value addFirst(Transaction tx, Key key, Value value) throws IOException { + if (key == null) { + throw new IllegalArgumentException("Key cannot be null"); + } + entries.addFirst(new KeyValueEntry(key, value)); + store(tx, true); + return null; + } + + private void store(Transaction tx, boolean addFirst) throws IOException { + try { + index.storeNode(tx, this, allowOverflow()); + } catch ( Transaction.PageOverflowIOException e ) { + // If we get an overflow + split(tx, addFirst); + } + } + + private boolean allowOverflow() { + return false; + } + + private void split(Transaction tx, boolean isAddFirst) throws IOException { + ListNode extension = index.createNode(tx, this); + if (isAddFirst) { + extension.setEntries(entries.getHead().splitAfter()); + extension.setNext(this.getNext()); + this.setNext(extension.getPageId()); + } else { + index.setTailPageId(extension.getPageId()); + this.setNext(extension.getPageId()); + extension.setEntries(entries.getTail().getPrevious().splitAfter()); + } + index.storeNode(tx, this, false); + extension.store(tx, isAddFirst); + } + + // called after a split + private void setEntries(LinkedNodeList> list) { + this.entries = list; + } + + public Value get(Transaction tx, Key key) throws IOException { + if (key == null) { + throw new IllegalArgumentException("Key cannot be null"); + } + Value result = null; + KeyValueEntry nextEntry = entries.getTail(); + while (nextEntry != null) { + if (nextEntry.getKey().equals(key)) { + result = nextEntry.getValue(); + break; + } + nextEntry = nextEntry.getPrevious(); + } + return result; + } + + public boolean isEmpty(final Transaction tx) throws IOException { + return entries.isEmpty(); + } + + public Entry getFirst(Transaction tx) throws IOException { + return entries.getHead(); + } + + public Entry getLast(Transaction tx) throws IOException { + return entries.getTail(); + } + + public Iterator> iterator(final Transaction tx, int pos) throws IOException { + return new ListIterator(tx, this, pos); + } + + public Iterator> iterator(final Transaction tx) throws IOException { + return new ListIterator(tx, this, 0); + } + + Iterator> listNodeIterator(final Transaction tx) throws IOException { + return new ListNodeIterator(tx, this); + } + + public void clear(Transaction tx) throws IOException { + entries.clear(); + tx.free(this.getPageId()); + } + + public boolean contains(Transaction tx, Key key) throws IOException { + if (key == null) { + throw new IllegalArgumentException("Key cannot be null"); + } + boolean found = false; + KeyValueEntry nextEntry = entries.getTail(); + while (nextEntry != null) { + if (nextEntry.getKey().equals(key)) { + found = true; + break; + } + nextEntry = nextEntry.getPrevious(); + } + return found; + } + + /////////////////////////////////////////////////////////////////// + // Implementation methods + /////////////////////////////////////////////////////////////////// + + public long getPageId() { + return page.getPageId(); + } + + public ListNode getParent() { + return parent; + } + + public void setParent(ListNode parent) { + this.parent = parent; + } + + public Page> getPage() { + return page; + } + + public void setPage(Page> page) { + this.page = page; + } + + public long getNext() { + return next; + } + + public void setNext(long next) { + this.next = next; + } + + @Override + public String toString() { + return "[ListNode "+ entries.toString() + "]"; + } +} + + diff --git a/kahadb/src/main/java/org/apache/kahadb/page/PageFile.java b/kahadb/src/main/java/org/apache/kahadb/page/PageFile.java index ba8f6d8239..c1e50fa264 100644 --- a/kahadb/src/main/java/org/apache/kahadb/page/PageFile.java +++ b/kahadb/src/main/java/org/apache/kahadb/page/PageFile.java @@ -634,7 +634,7 @@ public class PageFile { } /** - * @param allows you to enable read page caching + * @param enablePageCaching allows you to enable read page caching */ public void setEnablePageCaching(boolean enablePageCaching) { assertNotLoaded(); @@ -649,7 +649,7 @@ public class PageFile { } /** - * @param Sets the maximum number of pages that will get stored in the read page cache. + * @param pageCacheSize Sets the maximum number of pages that will get stored in the read page cache. */ public void setPageCacheSize(int pageCacheSize) { assertNotLoaded(); @@ -680,6 +680,11 @@ public class PageFile { return recoveryFileMinPageCount; } + public long getFreePageCount() { + assertLoaded(); + return freeList.size(); + } + public void setRecoveryFileMinPageCount(int recoveryFileMinPageCount) { assertNotLoaded(); this.recoveryFileMinPageCount = recoveryFileMinPageCount; @@ -915,8 +920,6 @@ public class PageFile { /** * - * @param timeout - * @param unit * @return true if there are still pending writes to do. * @throws InterruptedException * @throws IOException diff --git a/kahadb/src/main/java/org/apache/kahadb/util/LinkedNode.java b/kahadb/src/main/java/org/apache/kahadb/util/LinkedNode.java index f0e23ddbee..b7402d2214 100644 --- a/kahadb/src/main/java/org/apache/kahadb/util/LinkedNode.java +++ b/kahadb/src/main/java/org/apache/kahadb/util/LinkedNode.java @@ -262,8 +262,6 @@ public class LinkedNode> { // Update all the nodes in the new list so that they know of their new // list owner. T n = newList.head; - newList.size++; - list.size--; do { n.list = newList; n = n.next; @@ -301,8 +299,6 @@ public class LinkedNode> { // Update all the nodes in the new list so that they know of their new // list owner. T n = newList.head; - newList.size++; - list.size--; do { n.list = newList; n = n.next; diff --git a/kahadb/src/main/java/org/apache/kahadb/util/LinkedNodeList.java b/kahadb/src/main/java/org/apache/kahadb/util/LinkedNodeList.java index ec1e967fd7..976d52490f 100644 --- a/kahadb/src/main/java/org/apache/kahadb/util/LinkedNodeList.java +++ b/kahadb/src/main/java/org/apache/kahadb/util/LinkedNodeList.java @@ -48,7 +48,7 @@ public class LinkedNodeList> { } public T getTail() { - return head.prev; + return head != null ? head.prev : null; } public void clear() { diff --git a/kahadb/src/test/java/org/apache/kahadb/index/ListIndexTest.java b/kahadb/src/test/java/org/apache/kahadb/index/ListIndexTest.java new file mode 100644 index 0000000000..11d354fee3 --- /dev/null +++ b/kahadb/src/test/java/org/apache/kahadb/index/ListIndexTest.java @@ -0,0 +1,326 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kahadb.index; + +import java.text.NumberFormat; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; +import org.apache.kahadb.util.LongMarshaller; +import org.apache.kahadb.util.StringMarshaller; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ListIndexTest extends IndexTestSupport { + private static final Logger LOG = LoggerFactory.getLogger(ListIndexTest.class); + private NumberFormat nf; + + @Override + protected void setUp() throws Exception { + super.setUp(); + nf = NumberFormat.getIntegerInstance(); + nf.setMinimumIntegerDigits(6); + nf.setGroupingUsed(false); + } + + @Override + protected Index createIndex() throws Exception { + + long id = tx.allocate().getPageId(); + tx.commit(); + + ListIndex index = new ListIndex(pf, id); + index.setKeyMarshaller(StringMarshaller.INSTANCE); + index.setValueMarshaller(LongMarshaller.INSTANCE); + + return index; + } + + public void testSize() throws Exception { + createPageFileAndIndex(100); + + ListIndex listIndex = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + + int count = 30; + tx = pf.tx(); + doInsert(count); + tx.commit(); + assertEquals("correct size", count, listIndex.size()); + + tx = pf.tx(); + Iterator> iterator = index.iterator(tx); + while (iterator.hasNext()) { + iterator.next(); + iterator.remove(); + assertEquals("correct size", --count, listIndex.size()); + } + tx.commit(); + + count = 30; + tx = pf.tx(); + doInsert(count); + tx.commit(); + assertEquals("correct size", count, listIndex.size()); + + tx = pf.tx(); + listIndex.clear(tx); + assertEquals("correct size", 0, listIndex.size()); + tx.commit(); + } + + public void testAddFirst() throws Exception { + createPageFileAndIndex(100); + + ListIndex listIndex = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + + tx = pf.tx(); + + // put is add last + doInsert(10); + listIndex.addFirst(tx, key(10), (long) 10); + listIndex.addFirst(tx, key(11), (long) 11); + + tx.commit(); + + tx = pf.tx(); + int counter = 11; + Iterator> iterator = index.iterator(tx); + assertEquals(key(counter), iterator.next().getKey()); + counter--; + assertEquals(key(counter), iterator.next().getKey()); + counter--; + int count = 0; + while (iterator.hasNext() && count < counter) { + Map.Entry entry = (Map.Entry) iterator.next(); + assertEquals(key(count), entry.getKey()); + assertEquals(count, (long) entry.getValue()); + count++; + } + tx.commit(); + } + + public void testPruning() throws Exception { + createPageFileAndIndex(100); + + ListIndex index = ((ListIndex) this.index); + + this.index.load(tx); + tx.commit(); + + long pageCount = index.getPageFile().getPageCount(); + assertEquals(1, pageCount); + + long freePageCount = index.getPageFile().getFreePageCount(); + assertEquals("No free pages", 0, freePageCount); + + tx = pf.tx(); + doInsert(20); + tx.commit(); + + pageCount = index.getPageFile().getPageCount(); + LOG.info("page count: " + pageCount); + assertTrue("used some pages", pageCount > 1); + + tx = pf.tx(); + // Remove the data. + doRemove(20); + + tx.commit(); + + freePageCount = index.getPageFile().getFreePageCount(); + + LOG.info("FreePage count: " + freePageCount); + assertTrue("Some free pages " + freePageCount, freePageCount > 0); + + + LOG.info("add some more to use up free list"); + tx = pf.tx(); + doInsert(20); + tx.commit(); + + freePageCount = index.getPageFile().getFreePageCount(); + + LOG.info("FreePage count: " + freePageCount); + assertEquals("no free pages " + freePageCount, 0, freePageCount); + assertEquals("Page count is static", pageCount, index.getPageFile().getPageCount()); + + this.index.unload(tx); + tx.commit(); + } + + public void testIterationAddFirst() throws Exception { + createPageFileAndIndex(100); + ListIndex index = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + + tx = pf.tx(); + final int entryCount = 200; + // Insert in reverse order.. + doInsertReverse(entryCount); + this.index.unload(tx); + tx.commit(); + this.index.load(tx); + tx.commit(); + + int counter = 0; + for (Iterator> i = index.iterator(tx); i.hasNext(); ) { + Map.Entry entry = (Map.Entry) i.next(); + assertEquals(key(counter), entry.getKey()); + assertEquals(counter, (long) entry.getValue()); + counter++; + } + assertEquals("We iterated over all entries", entryCount, counter); + + tx = pf.tx(); + // Remove the data. + doRemove(entryCount); + tx.commit(); + + this.index.unload(tx); + tx.commit(); + } + + + public void testIteration() throws Exception { + createPageFileAndIndex(100); + ListIndex index = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + + // Insert in reverse order.. + final int entryCount = 200; + doInsert(entryCount); + + this.index.unload(tx); + tx.commit(); + this.index.load(tx); + tx.commit(); + + int counter = 0; + for (Iterator> i = index.iterator(tx); i.hasNext(); ) { + Map.Entry entry = (Map.Entry) i.next(); + assertEquals(key(counter), entry.getKey()); + assertEquals(counter, (long) entry.getValue()); + counter++; + } + assertEquals("We iterated over all entries", entryCount, counter); + + this.index.unload(tx); + tx.commit(); + } + + public void testVisitor() throws Exception { + createPageFileAndIndex(100); + ListIndex index = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + + // Insert in reverse order.. + doInsert(1000); + + this.index.unload(tx); + tx.commit(); + this.index.load(tx); + tx.commit(); + + // BTree should iterate it in sorted order. + + /*index.visit(tx, new BTreeVisitor(){ + public boolean isInterestedInKeysBetween(String first, String second) { + return true; + } + public void visit(List keys, List values) { + } + });*/ + + + this.index.unload(tx); + tx.commit(); + } + + + public void testRandomRemove() throws Exception { + + createPageFileAndIndex(100); + ListIndex index = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + + final int count = 4000; + doInsert(count); + + Random rand = new Random(System.currentTimeMillis()); + int i = 0, prev = 0; + while (!index.isEmpty(tx)) { + prev = i; + i = rand.nextInt(count); + try { + index.remove(tx, key(i)); + } catch (Exception e) { + e.printStackTrace(); + fail("unexpected exception on " + i + ", prev: " + prev + ", ex: " + e); + } + } + } + + public void testRemovePattern() throws Exception { + createPageFileAndIndex(100); + ListIndex index = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + + final int count = 4000; + doInsert(count); + + index.remove(tx, key(3697)); + index.remove(tx, key(1566)); + } + + public void testLargeAppendTimed() throws Exception { + createPageFileAndIndex(100); + ListIndex listIndex = ((ListIndex) this.index); + this.index.load(tx); + tx.commit(); + final int COUNT = 50000; + long start = System.currentTimeMillis(); + for (int i = 0; i < COUNT; i++) { + //String test = new String("test" + i); + //ByteSequence bs = new ByteSequence(test.getBytes()); + listIndex.put(tx, key(i), (long) i); + tx.commit(); + } + LOG.info("Time to add " + COUNT + ": " + (System.currentTimeMillis() - start) + " mills"); + LOG.info("Page count: " + listIndex.getPageFile().getPageCount()); + } + + void doInsertReverse(int count) throws Exception { + for (int i = count - 1; i >= 0; i--) { + ((ListIndex) index).addFirst(tx, key(i), (long) i); + tx.commit(); + } + } + + @Override + protected String key(int i) { + return "key:" + nf.format(i); + } +}