diff --git a/CHANGES.txt b/CHANGES.txt index 01e879f4a0f..91cf59ec8b4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -282,6 +282,14 @@ Bug fixes 22. LUCENE-1605: Added BitVector.subset(). (Jeremy Volkman via Mike McCandless) + +23. LUCENE-1618: Added FileSwitchDirectory that enables files with + specified extensions to be stored in a primary directory and the + rest of the files to be stored in the secondary directory. For + example, this can be useful for the large doc-store (stored + fields, term vectors) files in FSDirectory and the rest of the + index files in a RAMDirectory. (Jason Rutherglen via Mike + McCandless) 23. LUCENE-1494: Added FieldMaskingSpanQuery which can be used to cross-correlate Spans from different fields. diff --git a/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/src/java/org/apache/lucene/store/FileSwitchDirectory.java new file mode 100644 index 00000000000..027fe6c7681 --- /dev/null +++ b/src/java/org/apache/lucene/store/FileSwitchDirectory.java @@ -0,0 +1,127 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Files with the specified extensions are placed in the + * primary directory; others are placed in the secondary + * directory. The provided Set must not change once passed + * to this class, and must allow multiple threads to call + * contains at once. + */ + +public class FileSwitchDirectory extends Directory { + private final Directory secondaryDir; + private final Directory primaryDir; + private final Set primaryExtensions; + private boolean doClose; + + public FileSwitchDirectory(Set primaryExtensions, Directory primaryDir, Directory secondaryDir, boolean doClose) { + this.primaryExtensions = primaryExtensions; + this.primaryDir = primaryDir; + this.secondaryDir = secondaryDir; + this.doClose = doClose; + this.lockFactory = primaryDir.getLockFactory(); + } + + public void close() throws IOException { + if (doClose) { + try { + secondaryDir.close(); + } finally { + primaryDir.close(); + } + doClose = false; + } + } + + public String[] listAll() throws IOException { + List list = new ArrayList(); + String[] ramFiles = secondaryDir.listAll(); + for (int x = 0; x < ramFiles.length; x++) { + list.add(ramFiles[x]); + } + String[] fsFiles = primaryDir.listAll(); + for (int x = 0; x < fsFiles.length; x++) { + list.add(fsFiles[x]); + } + return (String[]) list.toArray(new String[0]); + } + + public String[] list() throws IOException { + return listAll(); + } + + public static String getExtension(String name) { + int i = name.lastIndexOf('.'); + if (i == -1) { + return ""; + } + return name.substring(i+1, name.length()); + } + + private Directory getDirectory(String name) { + String ext = getExtension(name); + if (primaryExtensions.contains(ext)) { + return primaryDir; + } else { + return secondaryDir; + } + } + + public boolean fileExists(String name) throws IOException { + return getDirectory(name).fileExists(name); + } + + public long fileModified(String name) throws IOException { + return getDirectory(name).fileModified(name); + } + + public void touchFile(String name) throws IOException { + getDirectory(name).touchFile(name); + } + + public void deleteFile(String name) throws IOException { + getDirectory(name).deleteFile(name); + } + + public void renameFile(String from, String to) throws IOException { + getDirectory(from).renameFile(from, to); + } + + public long fileLength(String name) throws IOException { + return getDirectory(name).fileLength(name); + } + + public IndexOutput createOutput(String name) throws IOException { + return getDirectory(name).createOutput(name); + } + + public void sync(String name) throws IOException { + getDirectory(name).sync(name); + } + + public IndexInput openInput(String name) throws IOException { + return getDirectory(name).openInput(name); + } +} diff --git a/src/test/org/apache/lucene/index/TestIndexWriterReader.java b/src/test/org/apache/lucene/index/TestIndexWriterReader.java index 81bee97816a..1633043880a 100644 --- a/src/test/org/apache/lucene/index/TestIndexWriterReader.java +++ b/src/test/org/apache/lucene/index/TestIndexWriterReader.java @@ -537,7 +537,6 @@ public class TestIndexWriterReader extends LuceneTestCase { public static void createIndexNoClose(boolean multiSegment, String indexName, IndexWriter w) throws IOException { - w.setMergePolicy(new LogDocMergePolicy()); for (int i = 0; i < 100; i++) { w.addDocument(createDocument(i, indexName, 4)); } diff --git a/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java b/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java new file mode 100644 index 00000000000..99f32b15056 --- /dev/null +++ b/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java @@ -0,0 +1,71 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.TestIndexWriterReader; +import org.apache.lucene.util.LuceneTestCase; + +public class TestFileSwitchDirectory extends LuceneTestCase { + /** + * Test if writing doc stores to disk and everything else to ram works. + * @throws IOException + */ + public void testBasic() throws IOException { + Set fileExtensions = new HashSet(); + fileExtensions.add("fdt"); + fileExtensions.add("fdx"); + + Directory primaryDir = new MockRAMDirectory(); + RAMDirectory secondaryDir = new MockRAMDirectory(); + + FileSwitchDirectory fsd = new FileSwitchDirectory(fileExtensions, primaryDir, secondaryDir, true); + IndexWriter writer = new IndexWriter(fsd, new WhitespaceAnalyzer(), + IndexWriter.MaxFieldLength.LIMITED); + writer.setUseCompoundFile(false); + TestIndexWriterReader.createIndexNoClose(true, "ram", writer); + IndexReader reader = writer.getReader(); + assertEquals(100, reader.maxDoc()); + writer.commit(); + // we should see only fdx,fdt files here + String[] files = primaryDir.listAll(); + assertTrue(files.length > 0); + for (int x=0; x < files.length; x++) { + String ext = FileSwitchDirectory.getExtension(files[x]); + assertTrue(fileExtensions.contains(ext)); + } + files = secondaryDir.listAll(); + assertTrue(files.length > 0); + // we should not see fdx,fdt files here + for (int x=0; x < files.length; x++) { + String ext = FileSwitchDirectory.getExtension(files[x]); + assertFalse(fileExtensions.contains(ext)); + } + reader.close(); + writer.close(); + + primaryDir.close(); + secondaryDir.close(); + } +}