LUCENE-1618: added FileSwitchDirectory, which will be used for NRT readers in IndexWriter

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@770841 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -282,6 +282,14 @@ Bug fixes
 
 22. LUCENE-1605: Added BitVector.subset().  (Jeremy Volkman via Mike
     McCandless)
+    
+23. LUCENE-1618: Added FileSwitchDirectory that enables files with
+    specified extensions to be stored in a primary directory and the
+    rest of the files to be stored in the secondary directory.  For
+    example, this can be useful for the large doc-store (stored
+    fields, term vectors) files in FSDirectory and the rest of the
+    index files in a RAMDirectory. (Jason Rutherglen via Mike
+    McCandless)
 
 23. LUCENE-1494: Added FieldMaskingSpanQuery which can be used to
     cross-correlate Spans from different fields.

src/java/org/apache/lucene/store/FileSwitchDirectory.java

@@ -0,0 +1,127 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Files with the specified extensions are placed in the
+ * primary directory; others are placed in the secondary
+ * directory.  The provided Set must not change once passed
+ * to this class, and must allow multiple threads to call
+ * contains at once.
+ */
+
+public class FileSwitchDirectory extends Directory {
+  private final Directory secondaryDir;
+  private final Directory primaryDir;
+  private final Set primaryExtensions;
+  private boolean doClose;
+
+  public FileSwitchDirectory(Set primaryExtensions, Directory primaryDir, Directory secondaryDir, boolean doClose) {
+    this.primaryExtensions = primaryExtensions;
+    this.primaryDir = primaryDir;
+    this.secondaryDir = secondaryDir;
+    this.doClose = doClose;
+    this.lockFactory = primaryDir.getLockFactory();
+  }
+  
+  public void close() throws IOException {
+    if (doClose) {
+      try {
+        secondaryDir.close();
+      } finally { 
+        primaryDir.close();
+      }
+      doClose = false;
+    }
+  }
+  
+  public String[] listAll() throws IOException {
+    List list = new ArrayList();
+    String[] ramFiles = secondaryDir.listAll();
+    for (int x = 0; x < ramFiles.length; x++) {
+      list.add(ramFiles[x]);
+    }
+    String[] fsFiles = primaryDir.listAll();
+    for (int x = 0; x < fsFiles.length; x++) {
+      list.add(fsFiles[x]);
+    }
+    return (String[]) list.toArray(new String[0]);
+  }
+  
+  public String[] list() throws IOException {
+    return listAll();
+  }
+
+  public static String getExtension(String name) {
+    int i = name.lastIndexOf('.');
+    if (i == -1) {
+      return "";
+    }
+    return name.substring(i+1, name.length());
+  }
+
+  private Directory getDirectory(String name) {
+    String ext = getExtension(name);
+    if (primaryExtensions.contains(ext)) {
+      return primaryDir;
+    } else {
+      return secondaryDir;
+    }
+  }
+
+  public boolean fileExists(String name) throws IOException {
+    return getDirectory(name).fileExists(name);
+  }
+
+  public long fileModified(String name) throws IOException {
+    return getDirectory(name).fileModified(name);
+  }
+
+  public void touchFile(String name) throws IOException {
+    getDirectory(name).touchFile(name);
+  }
+
+  public void deleteFile(String name) throws IOException {
+    getDirectory(name).deleteFile(name);
+  }
+
+  public void renameFile(String from, String to) throws IOException {
+    getDirectory(from).renameFile(from, to);
+  }
+
+  public long fileLength(String name) throws IOException {
+    return getDirectory(name).fileLength(name);
+  }
+
+  public IndexOutput createOutput(String name) throws IOException {
+    return getDirectory(name).createOutput(name);
+  }
+
+  public void sync(String name) throws IOException {
+    getDirectory(name).sync(name);
+  }
+
+  public IndexInput openInput(String name) throws IOException {
+    return getDirectory(name).openInput(name);
+  }
+}

src/test/org/apache/lucene/index/TestIndexWriterReader.java

@@ -537,7 +537,6 @@ public class TestIndexWriterReader extends LuceneTestCase {
 
   public static void createIndexNoClose(boolean multiSegment, String indexName,
       IndexWriter w) throws IOException {
-    w.setMergePolicy(new LogDocMergePolicy());
     for (int i = 0; i < 100; i++) {
       w.addDocument(createDocument(i, indexName, 4));
     }

src/test/org/apache/lucene/store/TestFileSwitchDirectory.java

@@ -0,0 +1,71 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.TestIndexWriterReader;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestFileSwitchDirectory extends LuceneTestCase {
+  /**
+   * Test if writing doc stores to disk and everything else to ram works.
+   * @throws IOException
+   */
+  public void testBasic() throws IOException {
+    Set fileExtensions = new HashSet();
+    fileExtensions.add("fdt");
+    fileExtensions.add("fdx");
+    
+    Directory primaryDir = new MockRAMDirectory();
+    RAMDirectory secondaryDir = new MockRAMDirectory();
+    
+    FileSwitchDirectory fsd = new FileSwitchDirectory(fileExtensions, primaryDir, secondaryDir, true);
+    IndexWriter writer = new IndexWriter(fsd, new WhitespaceAnalyzer(),
+        IndexWriter.MaxFieldLength.LIMITED);
+    writer.setUseCompoundFile(false);
+    TestIndexWriterReader.createIndexNoClose(true, "ram", writer);
+    IndexReader reader = writer.getReader();
+    assertEquals(100, reader.maxDoc());
+    writer.commit();
+    // we should see only fdx,fdt files here
+    String[] files = primaryDir.listAll();
+    assertTrue(files.length > 0);
+    for (int x=0; x < files.length; x++) {
+      String ext = FileSwitchDirectory.getExtension(files[x]);
+      assertTrue(fileExtensions.contains(ext));
+    }
+    files = secondaryDir.listAll();
+    assertTrue(files.length > 0);
+    // we should not see fdx,fdt files here
+    for (int x=0; x < files.length; x++) {
+      String ext = FileSwitchDirectory.getExtension(files[x]);
+      assertFalse(fileExtensions.contains(ext));
+    }
+    reader.close();
+    writer.close();
+    
+    primaryDir.close();
+    secondaryDir.close();
+  }
+}