From 899c0929745d20997e300b6e0f64eaf8bebb6d33 Mon Sep 17 00:00:00 2001 From: Erik Hatcher Date: Wed, 14 Jan 2004 00:29:45 +0000 Subject: [PATCH] Andi Vajda's Berkeley DB directory contribution git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150924 13f79535-47bb-0310-9956-ffa450edef68 --- sandbox/contributions/db/build.xml | 18 ++ sandbox/contributions/db/lib/LICENSE.txt | 94 ++++++ .../org/apache/lucene/store/db/Block.java | 131 ++++++++ .../apache/lucene/store/db/DbDirectory.java | 230 ++++++++++++++ .../apache/lucene/store/db/DbInputStream.java | 163 ++++++++++ .../org/apache/lucene/store/db/DbLock.java | 92 ++++++ .../lucene/store/db/DbOutputStream.java | 176 +++++++++++ .../java/org/apache/lucene/store/db/File.java | 290 ++++++++++++++++++ 8 files changed, 1194 insertions(+) create mode 100644 sandbox/contributions/db/build.xml create mode 100644 sandbox/contributions/db/lib/LICENSE.txt create mode 100644 sandbox/contributions/db/src/java/org/apache/lucene/store/db/Block.java create mode 100644 sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbDirectory.java create mode 100644 sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbInputStream.java create mode 100644 sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbLock.java create mode 100644 sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbOutputStream.java create mode 100644 sandbox/contributions/db/src/java/org/apache/lucene/store/db/File.java diff --git a/sandbox/contributions/db/build.xml b/sandbox/contributions/db/build.xml new file mode 100644 index 00000000000..9b021c51c76 --- /dev/null +++ b/sandbox/contributions/db/build.xml @@ -0,0 +1,18 @@ + + + + + Lucene DB integration + + + + + + + + + + diff --git a/sandbox/contributions/db/lib/LICENSE.txt b/sandbox/contributions/db/lib/LICENSE.txt new file mode 100644 index 00000000000..e37033a5f85 --- /dev/null +++ b/sandbox/contributions/db/lib/LICENSE.txt @@ -0,0 +1,94 @@ +* + * Copyright (c) 1990-2004 + * Sleepycat Software. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Redistributions in any form must be accompanied by information on + * how to obtain complete source code for the DB software and any + * accompanying software that uses the DB software. The source code + * must either be included in the distribution or be available for no + * more than the cost of distribution plus a nominal fee, and must be + * freely redistributable under reasonable conditions. For an + * executable file, complete source code means the source code for all + * modules it contains. It does not include source code for modules or + * files that typically accompany the major components of the operating + * system on which the executable file runs. + * + * THIS SOFTWARE IS PROVIDED BY SLEEPYCAT SOFTWARE ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR + * NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL SLEEPYCAT SOFTWARE + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * Copyright (c) 1990, 1993, 1994, 1995 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 1995, 1996 + * The President and Fellows of Harvard University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY HARVARD AND ITS CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL HARVARD OR ITS CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + + \ No newline at end of file diff --git a/sandbox/contributions/db/src/java/org/apache/lucene/store/db/Block.java b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/Block.java new file mode 100644 index 00000000000..6897bd1e46c --- /dev/null +++ b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/Block.java @@ -0,0 +1,131 @@ +package org.apache.lucene.store.db; + +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2004 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" and + * "Apache Lucene" must not be used to endorse or promote products + * derived from this software without prior written permission. For + * written permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * "Apache Lucene", nor may "Apache" appear in their name, without + * prior written permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by the Open Source + * Applications Foundation on behalf of the Apache Software Foundation. + * For more information on the Open Source Applications Foundation, please see + * . + * For more information on the Apache Software Foundation, please see + * . + */ + +import java.io.IOException; + +import com.sleepycat.db.Dbt; +import com.sleepycat.db.Db; +import com.sleepycat.db.DbTxn; +import com.sleepycat.db.DbException; + +/** + * @author Andi Vajda + */ + +public class Block extends Object { + protected Dbt key, data; + + protected Block(File file) + throws IOException + { + byte[] fileKey = file.getKey(); + + key = new Dbt(new byte[fileKey.length + 8]); + key.setFlags(Db.DB_DBT_USERMEM); + + data = new Dbt(new byte[DbOutputStream.BLOCK_LEN]); + data.setUserBufferLength(data.getSize()); + data.setFlags(Db.DB_DBT_USERMEM); + + System.arraycopy(fileKey, 0, key.getData(), 0, fileKey.length); + seek(0L); + } + + protected byte[] getKey() + { + return key.getData(); + } + + protected byte[] getData() + { + return data.getData(); + } + + protected void seek(long position) + throws IOException + { + position = position >>> DbOutputStream.BLOCK_SHIFT; + byte[] data = key.getData(); + int last = data.length - 1; + + for (int i = 0; i < 8; i++) { + data[last - i] = (byte) (position & 0xff); + position >>>= 8; + } + } + + protected void get(Db blocks, DbTxn txn) + throws IOException + { + try { + blocks.get(txn, key, data, 0); + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + } + + protected void put(Db blocks, DbTxn txn) + throws IOException + { + try { + blocks.put(txn, key, data, 0); + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + } +} diff --git a/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbDirectory.java b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbDirectory.java new file mode 100644 index 00000000000..2052f7ab827 --- /dev/null +++ b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbDirectory.java @@ -0,0 +1,230 @@ +package org.apache.lucene.store.db; + +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2004 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" and + * "Apache Lucene" must not be used to endorse or promote products + * derived from this software without prior written permission. For + * written permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * "Apache Lucene", nor may "Apache" appear in their name, without + * prior written permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by the Open Source + * Applications Foundation on behalf of the Apache Software Foundation. + * For more information on the Open Source Applications Foundation, please see + * . + * For more information on the Apache Software Foundation, please see + * . + */ + +import java.io.IOException; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.util.List; +import java.util.ArrayList; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.OutputStream; +import org.apache.lucene.store.InputStream; + +import com.sleepycat.db.DbEnv; +import com.sleepycat.db.Db; +import com.sleepycat.db.Dbt; +import com.sleepycat.db.Dbc; +import com.sleepycat.db.DbTxn; +import com.sleepycat.db.DbException; + +/** + * A DbDirectory is a Berkeley DB 4.2 based implementation of + * {@link org.apache.lucene.store.Directory Directory}. It uses two + * {@link com.sleepycat.db.Db Db} database handles, one for storing file + * records and another for storing file data blocks. + * + * @author Andi Vajda + */ + +public class DbDirectory extends Directory { + + protected Db files, blocks; + protected DbTxn txn; + + /** + * Instantiate a DbDirectory. The same threading rules that apply to + * Berkeley DB handles apply to instances of DbDirectory. + * + * @param txn a transaction handle that is going to be used for all db + * operations done by this instance. This parameter may be + * null. + * @param files a db handle to store file records. + * @param blocks a db handle to store file data blocks. + */ + + public DbDirectory(DbTxn txn, Db files, Db blocks) + { + super(); + + this.txn = txn; + this.files = files; + this.blocks = blocks; + } + + public void close() + throws IOException + { + } + + public OutputStream createFile(String name) + throws IOException + { + return new DbOutputStream(files, blocks, txn, name, true); + } + + public void deleteFile(String name) + throws IOException + { + new File(name).delete(files, blocks, txn); + } + + public boolean fileExists(String name) + throws IOException + { + return new File(name).exists(files, txn); + } + + public long fileLength(String name) + throws IOException + { + File file = new File(name); + + if (file.exists(files, txn)) + return file.getLength(); + + throw new IOException("File does not exist: " + name); + } + + public long fileModified(String name) + throws IOException + { + File file = new File(name); + + if (file.exists(files, txn)) + return file.getTimeModified(); + + throw new IOException("File does not exist: " + name); + } + + public String[] list() + throws IOException + { + Dbc cursor = null; + List list = new ArrayList(); + + try { + try { + Dbt key = new Dbt(new byte[0]); + Dbt data = new Dbt(null); + + data.setPartialLength(0); + data.setFlags(Db.DB_DBT_PARTIAL); + + cursor = files.cursor(txn, 0); + + if (cursor.get(key, data, Db.DB_SET_RANGE) != Db.DB_NOTFOUND) + { + ByteArrayInputStream buffer = + new ByteArrayInputStream(key.getData()); + DataInputStream in = new DataInputStream(buffer); + String name = in.readUTF(); + + in.close(); + list.add(name); + + while (cursor.get(key, data, Db.DB_NEXT) != Db.DB_NOTFOUND) { + buffer = new ByteArrayInputStream(key.getData()); + in = new DataInputStream(buffer); + name = in.readUTF(); + in.close(); + + list.add(name); + } + } + } finally { + if (cursor != null) + cursor.close(); + } + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + + return (String[]) list.toArray(new String[list.size()]); + } + + public InputStream openFile(String name) + throws IOException + { + return new DbInputStream(files, blocks, txn, name); + } + + public Lock makeLock(String name) + { + return new DbLock(); + } + + public void renameFile(String from, String to) + throws IOException + { + new File(from).rename(files, blocks, txn, to); + } + + public void touchFile(String name) + throws IOException + { + File file = new File(name); + long length = 0L; + + if (file.exists(files, txn)) + length = file.getLength(); + + file.modify(files, txn, length, System.currentTimeMillis()); + } +} diff --git a/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbInputStream.java b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbInputStream.java new file mode 100644 index 00000000000..0d88feef713 --- /dev/null +++ b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbInputStream.java @@ -0,0 +1,163 @@ +package org.apache.lucene.store.db; + +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2004 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" and + * "Apache Lucene" must not be used to endorse or promote products + * derived from this software without prior written permission. For + * written permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * "Apache Lucene", nor may "Apache" appear in their name, without + * prior written permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by the Open Source + * Applications Foundation on behalf of the Apache Software Foundation. + * For more information on the Open Source Applications Foundation, please see + * . + * For more information on the Apache Software Foundation, please see + * . + */ + +import java.io.IOException; + +import org.apache.lucene.store.InputStream; + +import com.sleepycat.db.Db; +import com.sleepycat.db.DbTxn; +import com.sleepycat.db.Dbt; +import com.sleepycat.db.DbException; + +/** + * @author Andi Vajda + */ + +public class DbInputStream extends InputStream { + + protected long position = 0L; + protected File file; + protected Block block; + protected DbTxn txn; + protected Db files, blocks; + + protected DbInputStream(Db files, Db blocks, DbTxn txn, String name) + throws IOException + { + super(); + + this.files = files; + this.blocks = blocks; + this.txn = txn; + + this.file = new File(name); + if (!file.exists(files, txn)) + throw new IOException("File does not exist: " + name); + + length = file.getLength(); + + block = new Block(file); + block.get(blocks, txn); + } + + public Object clone() + { + try { + DbInputStream clone = (DbInputStream) super.clone(); + + clone.block = new Block(file); + clone.block.seek(position); + clone.block.get(blocks, txn); + + return clone; + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + } + + public void close() + throws IOException + { + } + + protected void readInternal(byte[] b, int offset, int len) + throws IOException + { + int blockPos = (int) (position & DbOutputStream.BLOCK_MASK); + + if (position + len > length) + throw new IOException("Reading past end of file"); + + while (blockPos + len >= DbOutputStream.BLOCK_LEN) { + int blockLen = DbOutputStream.BLOCK_LEN - blockPos; + + System.arraycopy(block.getData(), blockPos, b, offset, blockLen); + + len -= blockLen; + offset += blockLen; + position += blockLen; + + block.seek(position); + block.get(blocks, txn); + blockPos = 0; + } + + if (len > 0) + { + System.arraycopy(block.getData(), blockPos, b, offset, len); + position += len; + } + } + + protected void seekInternal(long pos) + throws IOException + { + if (pos > length) + throw new IOException("seeking past end of file"); + + if ((pos >>> DbOutputStream.BLOCK_SHIFT) != + (position >>> DbOutputStream.BLOCK_SHIFT)) + { + block.seek(pos); + block.get(blocks, txn); + } + + position = pos; + } +} diff --git a/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbLock.java b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbLock.java new file mode 100644 index 00000000000..f53a57154b6 --- /dev/null +++ b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbLock.java @@ -0,0 +1,92 @@ +package org.apache.lucene.store.db; + +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2004 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" and + * "Apache Lucene" must not be used to endorse or promote products + * derived from this software without prior written permission. For + * written permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * "Apache Lucene", nor may "Apache" appear in their name, without + * prior written permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by the Open Source + * Applications Foundation on behalf of the Apache Software Foundation. + * For more information on the Open Source Applications Foundation, please see + * . + * For more information on the Apache Software Foundation, please see + * . + */ + +import org.apache.lucene.store.Lock; + +/** + * This implementation of {@link org.apache.lucene.store.Lock Lock} is + * trivial as {@link DbDirectory} operations are managed by the Berkeley DB + * locking system. + * + * @author Andi Vajda + */ + +public class DbLock extends Lock { + + boolean isLocked = false; + + public DbLock() + { + } + + public boolean obtain() + { + return (isLocked = true); + } + + public void release() + { + isLocked = false; + } + + public boolean isLocked() + { + return isLocked; + } +} + diff --git a/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbOutputStream.java b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbOutputStream.java new file mode 100644 index 00000000000..3f4af53c9f5 --- /dev/null +++ b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/DbOutputStream.java @@ -0,0 +1,176 @@ +package org.apache.lucene.store.db; + +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2004 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" and + * "Apache Lucene" must not be used to endorse or promote products + * derived from this software without prior written permission. For + * written permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * "Apache Lucene", nor may "Apache" appear in their name, without + * prior written permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by the Open Source + * Applications Foundation on behalf of the Apache Software Foundation. + * For more information on the Open Source Applications Foundation, please see + * . + * For more information on the Apache Software Foundation, please see + * . + */ + +import java.io.IOException; + +import org.apache.lucene.store.OutputStream; + +import com.sleepycat.db.Db; +import com.sleepycat.db.DbTxn; +import com.sleepycat.db.Dbt; +import com.sleepycat.db.DbException; + +/** + * @author Andi Vajda + */ + +public class DbOutputStream extends OutputStream { + + /** + * The size of data blocks, currently 16k (2^14), is determined by this + * constant. + */ + static public final int BLOCK_SHIFT = 14; + static public final int BLOCK_LEN = 1 << BLOCK_SHIFT; + static public final int BLOCK_MASK = BLOCK_LEN - 1; + + protected long position = 0L, length = 0L; + protected File file; + protected Block block; + protected DbTxn txn; + protected Db files, blocks; + + protected DbOutputStream(Db files, Db blocks, DbTxn txn, + String name, boolean create) + throws IOException + { + super(); + + this.files = files; + this.blocks = blocks; + this.txn = txn; + + file = new File(files, blocks, txn, name, create); + block = new Block(file); + length = file.getLength(); + + seek(length); + block.get(blocks, txn); + } + + public void close() + throws IOException + { + flush(); + if (length > 0) + block.put(blocks, txn); + + file.modify(files, txn, length, System.currentTimeMillis()); + } + + protected void flushBuffer(byte[] b, int len) + throws IOException + { + int blockPos = (int) (position & BLOCK_MASK); + int offset = 0; + + while (blockPos + len >= BLOCK_LEN) { + int blockLen = BLOCK_LEN - blockPos; + + System.arraycopy(b, offset, block.getData(), blockPos, blockLen); + block.put(blocks, txn); + + len -= blockLen; + offset += blockLen; + position += blockLen; + + block.seek(position); + block.get(blocks, txn); + blockPos = 0; + } + + if (len > 0) + { + System.arraycopy(b, offset, block.getData(), blockPos, len); + position += len; + } + + if (position > length) + length = position; + } + + public long length() + throws IOException + { + return length; + } + + public void seek(long pos) + throws IOException + { + super.seek(pos); + seekInternal(pos); + } + + protected void seekInternal(long pos) + throws IOException + { + if (pos > length) + throw new IOException("seeking past end of file"); + + if ((pos >>> BLOCK_SHIFT) == (position >>> BLOCK_SHIFT)) + position = pos; + else + { + block.put(blocks, txn); + block.seek(pos); + block.get(blocks, txn); + position = pos; + } + } +} diff --git a/sandbox/contributions/db/src/java/org/apache/lucene/store/db/File.java b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/File.java new file mode 100644 index 00000000000..ff4f1bddd59 --- /dev/null +++ b/sandbox/contributions/db/src/java/org/apache/lucene/store/db/File.java @@ -0,0 +1,290 @@ +package org.apache.lucene.store.db; + +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2004 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" and + * "Apache Lucene" must not be used to endorse or promote products + * derived from this software without prior written permission. For + * written permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * "Apache Lucene", nor may "Apache" appear in their name, without + * prior written permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by the Open Source + * Applications Foundation on behalf of the Apache Software Foundation. + * For more information on the Open Source Applications Foundation, please see + * . + * For more information on the Apache Software Foundation, please see + * . + */ + +import java.io.IOException; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.util.Random; + +import com.sleepycat.db.Dbt; +import com.sleepycat.db.Dbc; +import com.sleepycat.db.Db; +import com.sleepycat.db.DbTxn; +import com.sleepycat.db.DbException; + +/** + * @author Andi Vajda + */ + +public class File extends Object { + + static protected Random random = new Random(); + + protected Dbt key, data; + protected long length, timeModified; + protected String name; + protected byte[] uuid; + + protected File(String name) + throws IOException + { + setName(name); + + data = new Dbt(new byte[32]); + data.setUserBufferLength(data.getSize()); + data.setFlags(Db.DB_DBT_USERMEM); + } + + protected File(Db files, Db blocks, DbTxn txn, String name, boolean create) + throws IOException + { + this(name); + + if (!exists(files, txn)) + { + if (!create) + throw new IOException("File does not exist: " + name); + else + { + Dbt key = new Dbt(new byte[24]); + Dbt data = new Dbt(null); + + key.setFlags(Db.DB_DBT_USERMEM); + key.setUserBufferLength(24); + data.setPartialLength(0); + data.setFlags(Db.DB_DBT_USERMEM | Db.DB_DBT_PARTIAL); + + uuid = new byte[16]; + + try { + do { + /* generate a v.4 random-uuid unique to this db */ + random.nextBytes(uuid); + uuid[6] = (byte) ((byte) 0x40 | + (uuid[6] & (byte) 0x0f)); + uuid[8] = (byte) ((byte) 0x80 | + (uuid[8] & (byte) 0x3f)); + System.arraycopy(uuid, 0, key.getData(), 0, 16); + } while (blocks.get(txn, key, data, 0) != Db.DB_NOTFOUND); + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + } + } + else if (create) + length = 0L; + } + + protected String getName() + { + return name; + } + + private void setName(String name) + throws IOException + { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(128); + DataOutputStream out = new DataOutputStream(buffer); + + out.writeUTF(name); + out.close(); + + key = new Dbt(buffer.toByteArray()); + key.setUserBufferLength(key.getSize()); + key.setFlags(Db.DB_DBT_USERMEM); + + this.name = name; + } + + protected byte[] getKey() + throws IOException + { + if (uuid == null) + throw new IOException("Uninitialized file"); + + return uuid; + } + + protected long getLength() + { + return length; + } + + protected long getTimeModified() + { + return timeModified; + } + + protected boolean exists(Db files, DbTxn txn) + throws IOException + { + try { + if (files.get(txn, key, data, 0) == Db.DB_NOTFOUND) + return false; + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + + byte[] bytes = data.getData(); + ByteArrayInputStream buffer = new ByteArrayInputStream(bytes); + DataInputStream in = new DataInputStream(buffer); + + length = in.readLong(); + timeModified = in.readLong(); + in.close(); + + uuid = new byte[16]; + System.arraycopy(bytes, 16, uuid, 0, 16); + + return true; + } + + protected void modify(Db files, DbTxn txn, long length, long timeModified) + throws IOException + { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(32); + DataOutputStream out = new DataOutputStream(buffer); + + out.writeLong(length); + out.writeLong(timeModified); + out.write(getKey()); + out.close(); + + System.arraycopy(buffer.toByteArray(), 0, data.getData(), 0, 32); + + try { + files.put(txn, key, data, 0); + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + + this.length = length; + this.timeModified = timeModified; + } + + protected void delete(Db files, Db blocks, DbTxn txn) + throws IOException + { + if (!exists(files, txn)) + throw new IOException("File does not exist: " + getName()); + + Dbc cursor = null; + + try { + try { + byte[] bytes = getKey(); + int ulen = bytes.length + 8; + byte[] cursorBytes = new byte[ulen]; + Dbt cursorKey = new Dbt(cursorBytes); + Dbt cursorData = new Dbt(null); + + System.arraycopy(bytes, 0, cursorBytes, 0, bytes.length); + cursorKey.setUserBufferLength(ulen); + cursorKey.setFlags(Db.DB_DBT_USERMEM); + cursorData.setPartialLength(0); + cursorData.setFlags(Db.DB_DBT_PARTIAL); + + cursor = blocks.cursor(txn, 0); + + if (cursor.get(cursorKey, cursorData, + Db.DB_SET_RANGE) != Db.DB_NOTFOUND) + { + cursor.delete(0); + + while (cursor.get(cursorKey, cursorData, Db.DB_NEXT) != + Db.DB_NOTFOUND) { + for (int i = 0; i < bytes.length; i++) + if (bytes[i] != cursorBytes[i]) + return; + + cursor.delete(0); + } + } + + files.delete(txn, key, 0); + } finally { + if (cursor != null) + cursor.close(); + } + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + } + + protected void rename(Db files, Db blocks, DbTxn txn, String name) + throws IOException + { + if (!exists(files, txn)) + throw new IOException("File does not exist: " + getName()); + + File newFile = new File(name); + + if (newFile.exists(files, txn)) + newFile.delete(files, blocks, txn); + + try { + files.delete(txn, key, 0); + setName(name); + files.put(txn, key, data, 0); + } catch (DbException e) { + throw new IOException(e.getMessage()); + } + } +}