From c312ac9d5283dfe32b50c21bb04ef56324d50c20 Mon Sep 17 00:00:00 2001 From: Andreas Vajda Date: Thu, 5 Jan 2006 01:04:17 +0000 Subject: [PATCH] - integrated Aaron Donovan's port to Berkeley DB Java Edition - reorganized directory tree to accomodate multiple implementations git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@366041 13f79535-47bb-0310-9956-ffa450edef68 --- contrib/db/bdb-je/build.xml | 38 ++ .../org/apache/lucene/store/je/Block.java | 83 ++++ .../java/org/apache/lucene/store/je/File.java | 233 ++++++++++ .../apache/lucene/store/je/JEDirectory.java | 218 ++++++++++ .../apache/lucene/store/je/JEIndexInput.java | 134 ++++++ .../apache/lucene/store/je/JEIndexOutput.java | 141 ++++++ .../org/apache/lucene/store/je/JELock.java | 50 +++ .../apache/lucene/store/je/JEStoreTest.java | 407 ++++++++++++++++++ contrib/db/bdb/build.xml | 33 ++ .../com/sleepycat/db/DbHandleExtractor.java | 0 .../org/apache/lucene/store/db/Block.java | 2 +- .../apache/lucene/store/db/DbDirectory.java | 2 +- .../apache/lucene/store/db/DbIndexInput.java | 2 +- .../apache/lucene/store/db/DbIndexOutput.java | 2 +- .../org/apache/lucene/store/db/DbLock.java | 2 +- .../java/org/apache/lucene/store/db/File.java | 2 +- .../apache/lucene/store/db/DbStoreTest.java | 0 contrib/db/build.xml | 42 +- 18 files changed, 1360 insertions(+), 31 deletions(-) create mode 100644 contrib/db/bdb-je/build.xml create mode 100644 contrib/db/bdb-je/src/java/org/apache/lucene/store/je/Block.java create mode 100644 contrib/db/bdb-je/src/java/org/apache/lucene/store/je/File.java create mode 100644 contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java create mode 100644 contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexInput.java create mode 100644 contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexOutput.java create mode 100644 contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JELock.java create mode 100644 contrib/db/bdb-je/src/test/org/apache/lucene/store/je/JEStoreTest.java create mode 100644 contrib/db/bdb/build.xml rename contrib/db/{ => bdb}/src/java/com/sleepycat/db/DbHandleExtractor.java (100%) rename contrib/db/{ => bdb}/src/java/org/apache/lucene/store/db/Block.java (98%) rename contrib/db/{ => bdb}/src/java/org/apache/lucene/store/db/DbDirectory.java (99%) rename contrib/db/{ => bdb}/src/java/org/apache/lucene/store/db/DbIndexInput.java (98%) rename contrib/db/{ => bdb}/src/java/org/apache/lucene/store/db/DbIndexOutput.java (98%) rename contrib/db/{ => bdb}/src/java/org/apache/lucene/store/db/DbLock.java (95%) rename contrib/db/{ => bdb}/src/java/org/apache/lucene/store/db/File.java (99%) rename contrib/db/{ => bdb}/src/test/org/apache/lucene/store/db/DbStoreTest.java (100%) diff --git a/contrib/db/bdb-je/build.xml b/contrib/db/bdb-je/build.xml new file mode 100644 index 00000000000..68520b427ab --- /dev/null +++ b/contrib/db/bdb-je/build.xml @@ -0,0 +1,38 @@ + + + + + Lucene Berkeley DB Java Edition integration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/Block.java b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/Block.java new file mode 100644 index 00000000000..3421676b7c0 --- /dev/null +++ b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/Block.java @@ -0,0 +1,83 @@ +package org.apache.lucene.store.je; + +/** + * Copyright 2002-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.DatabaseException; + +/** + * Port of Andi Vajda's DbDirectory to Java Edition of Berkeley Database + * + * @author Aaron Donovan + */ + +public class Block extends Object { + protected DatabaseEntry key, data; + + protected Block(File file) throws IOException { + byte[] fileKey = file.getKey(); + + key = new DatabaseEntry(new byte[fileKey.length + 8]); + data = new DatabaseEntry(new byte[JEIndexOutput.BLOCK_LEN]); + + System.arraycopy(fileKey, 0, key.getData(), 0, fileKey.length); + seek(0L); + } + + protected byte[] getKey() { + return key.getData(); + } + + protected byte[] getData() { + return data.getData(); + } + + protected void seek(long position) throws IOException { + byte[] data = key.getData(); + int index = data.length - 8; + + position >>>= JEIndexOutput.BLOCK_SHIFT; + + data[index + 0] = (byte) (0xff & (position >>> 56)); + data[index + 1] = (byte) (0xff & (position >>> 48)); + data[index + 2] = (byte) (0xff & (position >>> 40)); + data[index + 3] = (byte) (0xff & (position >>> 32)); + data[index + 4] = (byte) (0xff & (position >>> 24)); + data[index + 5] = (byte) (0xff & (position >>> 16)); + data[index + 6] = (byte) (0xff & (position >>> 8)); + data[index + 7] = (byte) (0xff & (position >>> 0)); + } + + protected void get(JEDirectory directory) throws IOException { + try { + // TODO check LockMode + directory.blocks.get(directory.txn, key, data, null); + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + } + + protected void put(JEDirectory directory) throws IOException { + try { + directory.blocks.put(directory.txn, key, data); + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + } +} diff --git a/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/File.java b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/File.java new file mode 100644 index 00000000000..046b8fdc251 --- /dev/null +++ b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/File.java @@ -0,0 +1,233 @@ +package org.apache.lucene.store.je; + +/** + * Copyright 2002-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Random; + +import com.sleepycat.je.Cursor; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.OperationStatus; +import com.sleepycat.je.Transaction; + +/** + * Port of Andi Vajda's DbDirectory to Java Edition of Berkeley Database + * + * @author Aaron Donovan + */ + +public class File extends Object { + + static protected Random random = new Random(); + + protected DatabaseEntry key, data; + + protected long length, timeModified; + + protected String name; + + protected byte[] uuid; + + protected File(String name) throws IOException { + setName(name); + + data = new DatabaseEntry(new byte[32]); + } + + protected File(JEDirectory directory, String name, boolean create) + throws IOException { + this(name); + + if (!exists(directory)) { + if (!create) + throw new IOException("File does not exist: " + name); + else { + DatabaseEntry key = new DatabaseEntry(new byte[24]); + DatabaseEntry data = new DatabaseEntry(null); + Database blocks = directory.blocks; + Transaction txn = directory.txn; + + data.setPartial(true); + + uuid = new byte[16]; + + try { + do { + /* generate a v.4 random-uuid unique to this db */ + random.nextBytes(uuid); + uuid[6] = (byte) ((byte) 0x40 | (uuid[6] & (byte) 0x0f)); + uuid[8] = (byte) ((byte) 0x80 | (uuid[8] & (byte) 0x3f)); + System.arraycopy(uuid, 0, key.getData(), 0, 16); + // TODO check LockMode + } while (blocks.get(txn, key, data, null) != OperationStatus.NOTFOUND); + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + } + } else if (create) + length = 0L; + } + + protected String getName() { + return name; + } + + private void setName(String name) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(128); + DataOutputStream out = new DataOutputStream(buffer); + + out.writeUTF(name); + out.close(); + + key = new DatabaseEntry(buffer.toByteArray()); + this.name = name; + } + + protected byte[] getKey() throws IOException { + if (uuid == null) + throw new IOException("Uninitialized file"); + + return uuid; + } + + protected long getLength() { + return length; + } + + protected long getTimeModified() { + return timeModified; + } + + protected boolean exists(JEDirectory directory) throws IOException { + Database files = directory.files; + Transaction txn = directory.txn; + try { + // TODO check LockMode + if (files.get(txn, key, data, null) == OperationStatus.NOTFOUND) + return false; + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + + byte[] bytes = data.getData(); + ByteArrayInputStream buffer = new ByteArrayInputStream(bytes); + DataInputStream in = new DataInputStream(buffer); + + length = in.readLong(); + timeModified = in.readLong(); + in.close(); + + uuid = new byte[16]; + System.arraycopy(bytes, 16, uuid, 0, 16); + + return true; + } + + protected void modify(JEDirectory directory, long length, long timeModified) + throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(32); + DataOutputStream out = new DataOutputStream(buffer); + Database files = directory.files; + Transaction txn = directory.txn; + + out.writeLong(length); + out.writeLong(timeModified); + out.write(getKey()); + out.close(); + + System.arraycopy(buffer.toByteArray(), 0, data.getData(), 0, 32); + + try { + files.put(txn, key, data); + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + + this.length = length; + this.timeModified = timeModified; + } + + protected void delete(JEDirectory directory) throws IOException { + if (!exists(directory)) + throw new IOException("File does not exist: " + getName()); + + Cursor cursor = null; + + try { + try { + byte[] bytes = getKey(); + int ulen = bytes.length + 8; + byte[] cursorBytes = new byte[ulen]; + DatabaseEntry cursorKey = new DatabaseEntry(cursorBytes); + DatabaseEntry cursorData = new DatabaseEntry(null); + Database files = directory.files; + Database blocks = directory.blocks; + Transaction txn = directory.txn; + + System.arraycopy(bytes, 0, cursorBytes, 0, bytes.length); + + cursorData.setPartial(true); + + cursor = blocks.openCursor(txn, null); + + if (cursor.getSearchKey(cursorKey, cursorData, null) != OperationStatus.NOTFOUND) { + cursor.delete(); + + while (cursor.getNextDup(cursorKey, cursorData, null) != OperationStatus.NOTFOUND) { + cursor.delete(); + } + } + + files.delete(txn, key); + } finally { + if (cursor != null) + cursor.close(); + } + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + + } + + protected void rename(JEDirectory directory, String name) + throws IOException { + if (!exists(directory)) + throw new IOException("File does not exist: " + getName()); + + File newFile = new File(name); + + if (newFile.exists(directory)) + newFile.delete(directory); + + try { + Database files = directory.files; + Transaction txn = directory.txn; + + files.delete(txn, key); + setName(name); + files.put(txn, key, data); + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + } +} diff --git a/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java new file mode 100644 index 00000000000..5c52da8abf7 --- /dev/null +++ b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java @@ -0,0 +1,218 @@ +package org.apache.lucene.store.je; + +/** + * Copyright 2002-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Lock; + +import com.sleepycat.je.Cursor; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.OperationStatus; +import com.sleepycat.je.Transaction; + +/** + * Port of Andi Vajda's DbDirectory to to Java Edition of Berkeley Database + * + * A JEDirectory is a Berkeley DB JE based implementation of + * {@link org.apache.lucene.store.Directory Directory}. It uses two + * {@link com.sleepycat.je.Database Db} database handles, one for storing file + * records and another for storing file data blocks. + * + * @author Aaron Donovan + */ + +public class JEDirectory extends Directory { + + protected Set openFiles = Collections.synchronizedSet(new HashSet()); + + protected Database files, blocks; + + protected Transaction txn; + + protected int flags; + + /** + * Instantiate a DbDirectory. The same threading rules that apply to + * Berkeley DB handles apply to instances of DbDirectory. + * + * @param txn + * a transaction handle that is going to be used for all db + * operations done by this instance. This parameter may be + * null. + * @param files + * a db handle to store file records. + * @param blocks + * a db handle to store file data blocks. + * @param flags + * flags used for db read operations. + */ + + public JEDirectory(Transaction txn, Database files, Database blocks, + int flags) { + super(); + + this.txn = txn; + this.files = files; + this.blocks = blocks; + this.flags = flags; + } + + public JEDirectory(Transaction txn, Database files, Database blocks) { + this(txn, files, blocks, 0); + } + + public void close() throws IOException { + flush(); + } + + /** + * Flush the currently open files. After they have been flushed it is safe + * to commit the transaction without closing this DbDirectory instance + * first. + * + * @see setTransaction + */ + public void flush() throws IOException { + Iterator iterator = openFiles.iterator(); + + while (iterator.hasNext()) { + System.out + .println(((JEIndexOutput) iterator.next()).file.getName()); + // ((IndexOutput) iterator.next()).flush(); + } + } + + public IndexOutput createOutput(String name) throws IOException { + return new JEIndexOutput(this, name, true); + } + + public void deleteFile(String name) throws IOException { + new File(name).delete(this); + } + + public boolean fileExists(String name) throws IOException { + return new File(name).exists(this); + } + + public long fileLength(String name) throws IOException { + File file = new File(name); + + if (file.exists(this)) + return file.getLength(); + + throw new IOException("File does not exist: " + name); + } + + public long fileModified(String name) throws IOException { + File file = new File(name); + + if (file.exists(this)) + return file.getTimeModified(); + + throw new IOException("File does not exist: " + name); + } + + public String[] list() throws IOException { + Cursor cursor = null; + List list = new ArrayList(); + + try { + try { + DatabaseEntry key = new DatabaseEntry(new byte[0]); + DatabaseEntry data = new DatabaseEntry(null); + + data.setPartial(true); + // TODO see if cursor needs configuration + cursor = files.openCursor(txn, null); + // TODO see if LockMode should be set + if (cursor.getNext(key, data, null) != OperationStatus.NOTFOUND) { + ByteArrayInputStream buffer = new ByteArrayInputStream(key + .getData()); + DataInputStream in = new DataInputStream(buffer); + String name = in.readUTF(); + + in.close(); + list.add(name); + + while (cursor.getNext(key, data, null) != OperationStatus.NOTFOUND) { + buffer = new ByteArrayInputStream(key.getData()); + in = new DataInputStream(buffer); + name = in.readUTF(); + in.close(); + + list.add(name); + } + } + } finally { + if (cursor != null) + cursor.close(); + } + } catch (DatabaseException e) { + throw new IOException(e.getMessage()); + } + + return (String[]) list.toArray(new String[list.size()]); + } + + public IndexInput openInput(String name) throws IOException { + return new JEIndexInput(this, name); + } + + public Lock makeLock(String name) { + return new JELock(); + } + + public void renameFile(String from, String to) throws IOException { + new File(from).rename(this, to); + } + + public void touchFile(String name) throws IOException { + File file = new File(name); + long length = 0L; + + if (file.exists(this)) + length = file.getLength(); + + file.modify(this, length, System.currentTimeMillis()); + } + + /** + * Once a transaction handle was committed it is no longer valid. In order + * to continue using this JEDirectory instance after a commit, the + * transaction handle has to be replaced. + * + * @param txn + * the new transaction handle to use + */ + public void setTransaction(Transaction txn) { + this.txn = txn; + } +} diff --git a/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexInput.java b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexInput.java new file mode 100644 index 00000000000..0fa57519281 --- /dev/null +++ b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexInput.java @@ -0,0 +1,134 @@ +package org.apache.lucene.store.je; + +/** + * Copyright 2002-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.store.IndexInput; + +/** + * Port of Andi Vajda's DbDirectory to Java Edition of Berkeley Database + * + * @author Aaron Donovan + */ + +public class JEIndexInput extends IndexInput { + + protected long position = 0L, length = 0L; + + protected JEDirectory directory; + + protected Block block; + + protected File file; + + protected JEIndexInput(JEDirectory directory, String name) + throws IOException { + super(); + + this.directory = directory; + + this.file = new File(name); + if (!file.exists(directory)) + throw new IOException("File does not exist: " + name); + + length = file.getLength(); + + block = new Block(file); + block.get(directory); + } + + public Object clone() { + try { + JEIndexInput clone = (JEIndexInput) super.clone(); + + clone.block = new Block(file); + clone.block.seek(position); + clone.block.get(directory); + + return clone; + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + } + + public void close() throws IOException { + } + + public long length() { + return length; + } + + public byte readByte() throws IOException { + if (position + 1 > length) + throw new IOException(file.getName() + ": Reading past end of file"); + + int blockPos = (int) (position++ & JEIndexOutput.BLOCK_MASK); + byte b = block.getData()[blockPos]; + + if (blockPos + 1 == JEIndexOutput.BLOCK_LEN) { + block.seek(position); + block.get(directory); + } + + return b; + } + + public void readBytes(byte[] b, int offset, int len) throws IOException { + if (position + len > length) + throw new IOException("Reading past end of file"); + else { + int blockPos = (int) (position & JEIndexOutput.BLOCK_MASK); + + while (blockPos + len >= JEIndexOutput.BLOCK_LEN) { + int blockLen = JEIndexOutput.BLOCK_LEN - blockPos; + + System + .arraycopy(block.getData(), blockPos, b, offset, + blockLen); + + len -= blockLen; + offset += blockLen; + position += blockLen; + + block.seek(position); + block.get(directory); + blockPos = 0; + } + + if (len > 0) { + System.arraycopy(block.getData(), blockPos, b, offset, len); + position += len; + } + } + } + + public void seek(long pos) throws IOException { + if (pos > length) + throw new IOException("seeking past end of file"); + + if ((pos >>> JEIndexOutput.BLOCK_SHIFT) != (position >>> JEIndexOutput.BLOCK_SHIFT)) { + block.seek(pos); + block.get(directory); + } + + position = pos; + } + + public long getFilePointer() { + return position; + } +} diff --git a/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexOutput.java b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexOutput.java new file mode 100644 index 00000000000..5dab52f1f87 --- /dev/null +++ b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEIndexOutput.java @@ -0,0 +1,141 @@ +package org.apache.lucene.store.je; + +/** + * Copyright 2002-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; + +/** + * Port of Andi Vajda's DbDirectory to Java Edition of Berkeley Database + * + * @author Aaron Donovan + */ + +public class JEIndexOutput extends IndexOutput { + + /** + * The size of data blocks, currently 16k (2^14), is determined by this + * constant. + */ + static public final int BLOCK_SHIFT = 14; + + static public final int BLOCK_LEN = 1 << BLOCK_SHIFT; + + static public final int BLOCK_MASK = BLOCK_LEN - 1; + + protected long position = 0L, length = 0L; + + protected JEDirectory directory; + + protected Block block; + + protected File file; + + protected JEIndexOutput(JEDirectory directory, String name, boolean create) + throws IOException { + super(); + + this.directory = directory; + + file = new File(directory, name, create); + block = new Block(file); + length = file.getLength(); + + seek(length); + block.get(directory); + + directory.openFiles.add(this); + } + + public void close() throws IOException { + flush(); + file.modify(directory, length, System.currentTimeMillis()); + + directory.openFiles.remove(this); + } + + public void flush() throws IOException { + if (length > 0) + block.put(directory); + } + + public void writeByte(byte b) throws IOException { + int blockPos = (int) (position++ & BLOCK_MASK); + + block.getData()[blockPos] = b; + + if (blockPos + 1 == BLOCK_LEN) { + block.put(directory); + block.seek(position); + block.get(directory); + } + + if (position > length) + length = position; + } + + public void writeBytes(byte[] b, int len) throws IOException { + int blockPos = (int) (position & BLOCK_MASK); + int offset = 0; + + while (blockPos + len >= BLOCK_LEN) { + int blockLen = BLOCK_LEN - blockPos; + + System.arraycopy(b, offset, block.getData(), blockPos, blockLen); + block.put(directory); + + len -= blockLen; + offset += blockLen; + position += blockLen; + + block.seek(position); + block.get(directory); + blockPos = 0; + } + + if (len > 0) { + System.arraycopy(b, offset, block.getData(), blockPos, len); + position += len; + } + + if (position > length) + length = position; + } + + public long length() throws IOException { + return length; + } + + public void seek(long pos) throws IOException { + if (pos > length) + throw new IOException("seeking past end of file"); + + if ((pos >>> BLOCK_SHIFT) == (position >>> BLOCK_SHIFT)) + position = pos; + else { + block.put(directory); + block.seek(pos); + block.get(directory); + position = pos; + } + } + + public long getFilePointer() { + return position; + } +} diff --git a/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JELock.java b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JELock.java new file mode 100644 index 00000000000..f2988f164fd --- /dev/null +++ b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JELock.java @@ -0,0 +1,50 @@ +package org.apache.lucene.store.je; + +/** + * Copyright 2002-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Lock; + +/** + * Port of Andi Vajda's DbDirectory to Java Edition of Berkeley Database + * + * @author Aaron Donovan + */ + +public class JELock extends Lock { + + boolean isLocked = false; + + public JELock() + { + } + + public boolean obtain() + { + return (isLocked = true); + } + + public void release() + { + isLocked = false; + } + + public boolean isLocked() + { + return isLocked; + } +} + diff --git a/contrib/db/bdb-je/src/test/org/apache/lucene/store/je/JEStoreTest.java b/contrib/db/bdb-je/src/test/org/apache/lucene/store/je/JEStoreTest.java new file mode 100644 index 00000000000..df5dd74b0cb --- /dev/null +++ b/contrib/db/bdb-je/src/test/org/apache/lucene/store/je/JEStoreTest.java @@ -0,0 +1,407 @@ +package org.apache.lucene.store.je; + +/** + * Copyright 2005 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.Random; + +import junit.framework.TestCase; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.je.JEDirectory; + +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; +import com.sleepycat.je.Transaction; + +/** + * Tests {@link JEDirectory}. + * + * Adapted from Andi Vajda's org.apache.lucene.db.DbStoreTest. + * @author Aaron Donovan + */ +public class JEStoreTest extends TestCase { + protected File dbHome = new File("index"); + protected Environment env; + protected Database index, blocks; + + public void setUp() + throws Exception + { + if (!dbHome.exists()) + dbHome.mkdir(); + else { + File[] files = dbHome.listFiles(); + + for (int i = 0; i < files.length; i++) { + String name = files[i].getName(); + if (name.endsWith("jdb") || name.equals("je.lck")) + files[i].delete(); + } + } + + EnvironmentConfig envConfig = new EnvironmentConfig(); + DatabaseConfig dbConfig = new DatabaseConfig(); + + envConfig.setTransactional(true); + envConfig.setAllowCreate(true); + dbConfig.setAllowCreate(true); + dbConfig.setTransactional(true); + + env = new Environment(dbHome, envConfig); + + Transaction txn = null; + + try { + txn = env.beginTransaction(null, null); + index = env.openDatabase(txn, "__index__", dbConfig); + blocks = env.openDatabase(txn, "__blocks__", dbConfig); + } catch (DatabaseException e) { + if (txn != null) + { + txn.abort(); + txn = null; + } + index = null; + blocks = null; + throw e; + } finally { + if (txn != null) + txn.commit(); + txn = null; + } + } + + public void tearDown() + throws Exception + { + if (index != null) + index.close(); + if (blocks != null) + blocks.close(); + if (env != null) + env.close(); + } + + public void testBytes() + throws Exception + { + final int count = 250; + final int LENGTH_MASK = 0xffff; + + Random gen = new Random(1251971); + int totalLength = 0; + int duration; + Date end; + + Date veryStart = new Date(); + Date start = new Date(); + Transaction txn = null; + Directory store = null; + + System.out.println("Writing files byte by byte"); + + try { + txn = env.beginTransaction(null, null); + store = new JEDirectory(txn, index, blocks); + + for (int i = 0; i < count; i++) { + String name = i + ".dat"; + int length = gen.nextInt() & LENGTH_MASK; + IndexOutput file = store.createOutput(name); + + totalLength += length; + + for (int j = 0; j < length; j++) { + byte b = (byte)(gen.nextInt() & 0x7F); + file.writeByte(b); + } + + file.close(); + } + } catch (IOException e) { + txn.abort(); + txn = null; + throw e; + } catch (DatabaseException e) { + if (txn != null) + { + txn.abort(); + txn = null; + } + throw e; + } finally { + if (txn != null) + txn.commit(); + + store.close(); + } + + end = new Date(); + + duration = (int) (end.getTime() - start.getTime()); + System.out.print(duration); + System.out.print(" total milliseconds to create, "); + System.out.print(totalLength / duration); + System.out.println(" kb/s"); + + try { + txn = env.beginTransaction(null, null); + store = new JEDirectory(txn, index, blocks); + + gen = new Random(1251971); + start = new Date(); + + for (int i = 0; i < count; i++) { + String name = i + ".dat"; + int length = gen.nextInt() & LENGTH_MASK; + IndexInput file = store.openInput(name); + + if (file.length() != length) + throw new Exception("length incorrect"); + + for (int j = 0; j < length; j++) { + byte b = (byte)(gen.nextInt() & 0x7F); + + if (file.readByte() != b) + throw new Exception("contents incorrect"); + } + + file.close(); + } + } catch (IOException e) { + txn.abort(); + txn = null; + throw e; + } catch (DatabaseException e) { + if (txn != null) + { + txn.abort(); + txn = null; + } + throw e; + } finally { + if (txn != null) + txn.commit(); + + store.close(); + } + + end = new Date(); + + duration = (int) (end.getTime() - start.getTime()); + System.out.print(duration); + System.out.print(" total milliseconds to read, "); + System.out.print(totalLength / duration); + System.out.println(" kb/s"); + + try { + txn = env.beginTransaction(null, null); + store = new JEDirectory(txn, index, blocks); + + gen = new Random(1251971); + start = new Date(); + + for (int i = 0; i < count; i++) { + String name = i + ".dat"; + store.deleteFile(name); + } + } catch (IOException e) { + txn.abort(); + txn = null; + throw e; + } catch (DatabaseException e) { + if (txn != null) + { + txn.abort(); + txn = null; + } + throw e; + } finally { + if (txn != null) + txn.commit(); + + store.close(); + } + + end = new Date(); + + System.out.print(end.getTime() - start.getTime()); + System.out.println(" total milliseconds to delete"); + + System.out.print(end.getTime() - veryStart.getTime()); + System.out.println(" total milliseconds"); + } + + public void testArrays() + throws Exception + { + final int count = 250; + final int LENGTH_MASK = 0xffff; + + Random gen = new Random(1251971); + int totalLength = 0; + int duration; + Date end; + + Date veryStart = new Date(); + Date start = new Date(); + Transaction txn = null; + Directory store = null; + + System.out.println("Writing files as one byte array"); + + try { + txn = env.beginTransaction(null, null); + store = new JEDirectory(txn, index, blocks); + + for (int i = 0; i < count; i++) { + String name = i + ".dat"; + int length = gen.nextInt() & LENGTH_MASK; + IndexOutput file = store.createOutput(name); + byte[] data = new byte[length]; + + totalLength += length; + gen.nextBytes(data); + file.writeBytes(data, length); + + file.close(); + } + } catch (IOException e) { + txn.abort(); + txn = null; + throw e; + } catch (DatabaseException e) { + if (txn != null) + { + txn.abort(); + txn = null; + } + throw e; + } finally { + if (txn != null) + txn.commit(); + + store.close(); + } + + end = new Date(); + + duration = (int) (end.getTime() - start.getTime()); + System.out.print(duration); + System.out.print(" total milliseconds to create, "); + System.out.print(totalLength / duration); + System.out.println(" kb/s"); + + try { + txn = env.beginTransaction(null, null); + store = new JEDirectory(txn, index, blocks); + + gen = new Random(1251971); + start = new Date(); + + for (int i = 0; i < count; i++) { + String name = i + ".dat"; + int length = gen.nextInt() & LENGTH_MASK; + IndexInput file = store.openInput(name); + + if (file.length() != length) + throw new Exception("length incorrect"); + + byte[] data = new byte[length]; + byte[] read = new byte[length]; + gen.nextBytes(data); + file.readBytes(read, 0, length); + + if (!Arrays.equals(data, read)) + throw new Exception("contents incorrect"); + + file.close(); + } + } catch (IOException e) { + txn.abort(); + txn = null; + throw e; + } catch (DatabaseException e) { + if (txn != null) + { + txn.abort(); + txn = null; + } + throw e; + } finally { + if (txn != null) + txn.commit(); + + store.close(); + } + + end = new Date(); + + duration = (int) (end.getTime() - start.getTime()); + System.out.print(duration); + System.out.print(" total milliseconds to read, "); + System.out.print(totalLength / duration); + System.out.println(" kb/s"); + + try { + txn = env.beginTransaction(null, null); + store = new JEDirectory(txn, index, blocks); + + gen = new Random(1251971); + start = new Date(); + + for (int i = 0; i < count; i++) { + String name = i + ".dat"; + store.deleteFile(name); + } + } catch (IOException e) { + txn.abort(); + txn = null; + throw e; + } catch (DatabaseException e) { + if (txn != null) + { + txn.abort(); + txn = null; + } + throw e; + } finally { + if (txn != null) + txn.commit(); + + store.close(); + } + + end = new Date(); + + System.out.print(end.getTime() - start.getTime()); + System.out.println(" total milliseconds to delete"); + + System.out.print(end.getTime() - veryStart.getTime()); + System.out.println(" total milliseconds"); + } +} + diff --git a/contrib/db/bdb/build.xml b/contrib/db/bdb/build.xml new file mode 100644 index 00000000000..dccfd907c7a --- /dev/null +++ b/contrib/db/bdb/build.xml @@ -0,0 +1,33 @@ + + + + + Lucene Berkeley DB integration + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/db/src/java/com/sleepycat/db/DbHandleExtractor.java b/contrib/db/bdb/src/java/com/sleepycat/db/DbHandleExtractor.java similarity index 100% rename from contrib/db/src/java/com/sleepycat/db/DbHandleExtractor.java rename to contrib/db/bdb/src/java/com/sleepycat/db/DbHandleExtractor.java diff --git a/contrib/db/src/java/org/apache/lucene/store/db/Block.java b/contrib/db/bdb/src/java/org/apache/lucene/store/db/Block.java similarity index 98% rename from contrib/db/src/java/org/apache/lucene/store/db/Block.java rename to contrib/db/bdb/src/java/org/apache/lucene/store/db/Block.java index 96d9f9c84d6..5d436a77e2b 100644 --- a/contrib/db/src/java/org/apache/lucene/store/db/Block.java +++ b/contrib/db/bdb/src/java/org/apache/lucene/store/db/Block.java @@ -1,7 +1,7 @@ package org.apache.lucene.store.db; /** - * Copyright 2002-2005 The Apache Software Foundation + * Copyright 2002-2006 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbDirectory.java similarity index 99% rename from contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java rename to contrib/db/bdb/src/java/org/apache/lucene/store/db/DbDirectory.java index 4d816743110..257ab3e25b5 100644 --- a/contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java +++ b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbDirectory.java @@ -1,7 +1,7 @@ package org.apache.lucene.store.db; /** - * Copyright 2002-2005 The Apache Software Foundation + * Copyright 2002-2006 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbIndexInput.java similarity index 98% rename from contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java rename to contrib/db/bdb/src/java/org/apache/lucene/store/db/DbIndexInput.java index bda325d3302..4d206732bb1 100644 --- a/contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java +++ b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbIndexInput.java @@ -1,7 +1,7 @@ package org.apache.lucene.store.db; /** - * Copyright 2002-2005 The Apache Software Foundation + * Copyright 2002-2006 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbIndexOutput.java similarity index 98% rename from contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java rename to contrib/db/bdb/src/java/org/apache/lucene/store/db/DbIndexOutput.java index fe601c36990..740c9a29326 100644 --- a/contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java +++ b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbIndexOutput.java @@ -1,7 +1,7 @@ package org.apache.lucene.store.db; /** - * Copyright 2002-2005 The Apache Software Foundation + * Copyright 2002-2006 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/contrib/db/src/java/org/apache/lucene/store/db/DbLock.java b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbLock.java similarity index 95% rename from contrib/db/src/java/org/apache/lucene/store/db/DbLock.java rename to contrib/db/bdb/src/java/org/apache/lucene/store/db/DbLock.java index 5faf9ebc44c..3c8f8c1df96 100644 --- a/contrib/db/src/java/org/apache/lucene/store/db/DbLock.java +++ b/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbLock.java @@ -1,7 +1,7 @@ package org.apache.lucene.store.db; /** - * Copyright 2002-2005 The Apache Software Foundation + * Copyright 2002-2006 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/contrib/db/src/java/org/apache/lucene/store/db/File.java b/contrib/db/bdb/src/java/org/apache/lucene/store/db/File.java similarity index 99% rename from contrib/db/src/java/org/apache/lucene/store/db/File.java rename to contrib/db/bdb/src/java/org/apache/lucene/store/db/File.java index b441ae4b248..271ed33a933 100644 --- a/contrib/db/src/java/org/apache/lucene/store/db/File.java +++ b/contrib/db/bdb/src/java/org/apache/lucene/store/db/File.java @@ -1,7 +1,7 @@ package org.apache.lucene.store.db; /** - * Copyright 2002-2005 The Apache Software Foundation + * Copyright 2002-2006 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/contrib/db/src/test/org/apache/lucene/store/db/DbStoreTest.java b/contrib/db/bdb/src/test/org/apache/lucene/store/db/DbStoreTest.java similarity index 100% rename from contrib/db/src/test/org/apache/lucene/store/db/DbStoreTest.java rename to contrib/db/bdb/src/test/org/apache/lucene/store/db/DbStoreTest.java diff --git a/contrib/db/build.xml b/contrib/db/build.xml index 0dd70a58c12..350cb8e778f 100644 --- a/contrib/db/build.xml +++ b/contrib/db/build.xml @@ -3,36 +3,28 @@ Lucene DB integration + - bdb: using the Java interface of C Berkeley DB + - bdb-je: using Berkeley DB Java Edition - - - - - - - - - - - - - - - - - + + - + + - - + + + + + + + + + + +