LUCENE-3807: remove file based store/load from Lookup

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1298049 13f79535-47bb-0310-9956-ffa450edef68

modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java

@@ -150,21 +150,4 @@ public abstract class Lookup {
    */
   public abstract boolean load(InputStream input) throws IOException;
   
-  /**
-   * Persist the constructed lookup data to a directory. Optional operation.
-   * @param storeDir directory where data can be stored.
-   * @return true if successful, false if unsuccessful or not supported.
-   * @throws IOException when fatal IO error occurs.
-   */
-  public abstract boolean store(File storeDir) throws IOException;
-
-  /**
-   * Discard current lookup data and load it from a previously saved copy.
-   * Optional operation.
-   * @param storeDir directory where lookup data was stored.
-   * @return true if completed successfully, false if unsuccessful or not supported.
-   * @throws IOException when fatal IO error occurs.
-   */
-  public abstract boolean load(File storeDir) throws IOException;
-  
 }

modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java

@@ -79,14 +79,6 @@ public class FSTCompletionLookup extends Lookup {
    */
   private final static int sharedTailLength = 5;
 
-  /**
-   * File name for the automaton.
-   * 
-   * @see #store(File)
-   * @see #load(File)
-   */
-  private static final String FILENAME = "fst.bin";
-
   private int buckets;
   private boolean exactMatchFirst;
 
@@ -264,46 +256,13 @@ public class FSTCompletionLookup extends Lookup {
     return bucket == -1 ? null : Long.valueOf(bucket);
   }
 
-  /**
-   * Deserialization from disk.
-   */
-  @Override
-  public synchronized boolean load(File storeDir) throws IOException {
-    File data = new File(storeDir, FILENAME);
-    if (!data.exists() || !data.canRead()) {
-      return false;
-    }
-
-    this.higherWeightsCompletion = new FSTCompletion(
-        FST.read(data, NoOutputs.getSingleton()));
-    this.normalCompletion = new FSTCompletion(
-        higherWeightsCompletion.getFST(), false, exactMatchFirst);
-
-    return true;
-  }
-
-  /**
-   * Serialization to disk.
-   */
-  @Override
-  public synchronized boolean store(File storeDir) throws IOException {
-    if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
-      return false;
-    }
-
-    if (this.normalCompletion == null) 
-      return false;
-
-    normalCompletion.getFST().save(new File(storeDir, FILENAME));
-    return true;
-  }
 
   @Override
   public synchronized boolean store(OutputStream output) throws IOException {
 
-    if (this.normalCompletion == null) 
-      return false;
     try {
+      if (this.normalCompletion == null || normalCompletion.getFST() == null) 
+        return false;
       normalCompletion.getFST().save(new OutputStreamDataOutput(output));
     } finally {
       IOUtils.close(output);

modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java

@@ -17,7 +17,6 @@ package org.apache.lucene.search.suggest.fst;
  * limitations under the License.
  */
 
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -63,14 +62,6 @@ import org.apache.lucene.util.fst.Util.MinResult;
  */
 public class WFSTCompletionLookup extends Lookup {
   
-  /**
-   * File name for the automaton.
-   * 
-   * @see #store(File)
-   * @see #load(File)
-   */
-  private static final String FILENAME = "wfst.bin";
-  
   /**
    * FST<Long>, weights are encoded as costs: (Integer.MAX_VALUE-weight)
    */
@@ -127,21 +118,13 @@ public class WFSTCompletionLookup extends Lookup {
     fst = builder.finish();
   }
 
-  @Override
-  public boolean store(File storeDir) throws IOException {
-    fst.save(new File(storeDir, FILENAME));
-    return true;
-  }
-
-  @Override
-  public boolean load(File storeDir) throws IOException {
-    this.fst = FST.read(new File(storeDir, FILENAME), PositiveIntOutputs.getSingleton(true));
-    return true;
-  }
   
   @Override
   public boolean store(OutputStream output) throws IOException {
     try {
+      if (fst == null) {
+        return false;
+      }
       fst.save(new OutputStreamDataOutput(output));
     } finally {
       IOUtils.close(output);

modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java

@@ -19,9 +19,6 @@ package org.apache.lucene.search.suggest.jaspell;
 
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -109,22 +106,11 @@ public class JaspellLookup extends Lookup {
     return res;
   }
 
-  public static final String FILENAME = "jaspell.dat";
   private static final byte LO_KID = 0x01;
   private static final byte EQ_KID = 0x02;
   private static final byte HI_KID = 0x04;
   private static final byte HAS_VALUE = 0x08;
  
-  
-  @Override
-  public boolean load(File storeDir) throws IOException {
-    File data = new File(storeDir, FILENAME);
-    if (!data.exists() || !data.canRead()) {
-      return false;
-    }
-    return load(new FileInputStream(data));
-  }
-  
   private void readRecursively(DataInputStream in, TSTNode node) throws IOException {
     node.splitchar = in.readChar();
     byte mask = in.readByte();
@@ -148,15 +134,6 @@ public class JaspellLookup extends Lookup {
     }
   }
 
-  @Override
-  public boolean store(File storeDir) throws IOException {
-    if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
-      return false;
-    }
-    File data = new File(storeDir, FILENAME);
-    return store(new FileOutputStream(data));
-  }
-  
   private void writeRecursively(DataOutputStream out, TSTNode node) throws IOException {
     if (node == null) {
       return;

modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java

@@ -119,23 +119,12 @@ public class TSTLookup extends Lookup {
     return res;
   }
   
-  public static final String FILENAME = "tst.dat";
-  
   private static final byte LO_KID = 0x01;
   private static final byte EQ_KID = 0x02;
   private static final byte HI_KID = 0x04;
   private static final byte HAS_TOKEN = 0x08;
   private static final byte HAS_VALUE = 0x10;
 
-  @Override
-  public synchronized boolean load(File storeDir) throws IOException {
-    File data = new File(storeDir, FILENAME);
-    if (!data.exists() || !data.canRead()) {
-      return false;
-    }
-    return load(new FileInputStream(data));
-  }
-  
   // pre-order traversal
   private void readRecursively(DataInputStream in, TernaryTreeNode node) throws IOException {
     node.splitchar = in.readChar();
@@ -160,15 +149,6 @@ public class TSTLookup extends Lookup {
     }
   }
 
-  @Override
-  public synchronized boolean store(File storeDir) throws IOException {
-    if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
-      return false;
-    }
-    File data = new File(storeDir, FILENAME);
-    return store(new FileOutputStream(data));
-  }
-  
   // pre-order traversal
   private void writeRecursively(DataOutputStream out, TernaryTreeNode node) throws IOException {
     // write out the current node

modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java

@@ -17,6 +17,8 @@
 package org.apache.lucene.search.suggest;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.util.List;
 
 import org.apache.lucene.search.suggest.Lookup;
@@ -69,11 +71,11 @@ public class PersistenceTest extends LuceneTestCase {
 
     // Store the suggester.
     File storeDir = TEMP_DIR;
-    lookup.store(storeDir);
+    lookup.store(new FileOutputStream(new File(storeDir, "lookup.dat")));
 
     // Re-read it from disk.
     lookup = lookupClass.newInstance();
-    lookup.load(storeDir);
+    lookup.load(new FileInputStream(new File(storeDir, "lookup.dat")));
 
     // Assert validity.
     long previous = Long.MIN_VALUE;

solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java

@@ -26,4 +26,5 @@ import org.apache.solr.core.SolrCore;
  */
 public abstract class LookupFactory {
   public abstract Lookup create(NamedList params, SolrCore core);
+  public abstract String storeFileName();
 }

solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java

@@ -18,6 +18,8 @@
 package org.apache.solr.spelling.suggest;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
@@ -73,6 +75,8 @@ public class Suggester extends SolrSpellChecker {
   protected Lookup lookup;
   protected String lookupImpl;
   protected SolrCore core;
+
+  private LookupFactory factory;
   
   @Override
   public String init(NamedList config, SolrCore core) {
@@ -92,7 +96,8 @@ public class Suggester extends SolrSpellChecker {
       lookupImpl = FSTLookupFactory.class.getName();
     }
 
-    LookupFactory factory = (LookupFactory) core.getResourceLoader().newInstance(lookupImpl);
+    factory = (LookupFactory) core.getResourceLoader().newInstance(lookupImpl);
+    
     lookup = factory.create(config, core);
     String store = (String)config.get(STORE_DIR);
     if (store != null) {
@@ -105,7 +110,7 @@ public class Suggester extends SolrSpellChecker {
       } else {
         // attempt reload of the stored lookup
         try {
-          lookup.load(storeDir);
+          lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())));
         } catch (IOException e) {
           LOG.warn("Loading stored lookup data failed", e);
         }
@@ -132,8 +137,19 @@ public class Suggester extends SolrSpellChecker {
     try {
       lookup.build(dictionary);
       if (storeDir != null) {
-        lookup.store(storeDir);
+        File target = new File(storeDir, factory.storeFileName());
+        if(!lookup.store(new FileOutputStream(target))) {
+          if (sourceLocation == null) {
+            assert reader != null && field != null;
+            LOG.error("Store Lookup build from index on field: " + field + " failed reader has: " + reader.maxDoc() + " docs");
+          } else {
+            LOG.error("Store Lookup build from sourceloaction: " + sourceLocation + " failed");
+          }
+        } else {
+          LOG.info("Stored suggest data to: " + target.getAbsolutePath());
+        }
       }
+
     } catch (Exception e) {
       LOG.error("Error while building or storing Suggester data", e);
     }
@@ -144,7 +160,7 @@ public class Suggester extends SolrSpellChecker {
     LOG.info("reload()");
     if (dictionary == null && storeDir != null) {
       // this may be a firstSearcher event, try loading it
-      if (lookup.load(storeDir)) {
+      if (lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())))) {
         return;  // loaded ok
       }
       LOG.debug("load failed, need to build Lookup again");

solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java

@@ -18,7 +18,7 @@ package org.apache.solr.spelling.suggest.fst;
  */
 
 import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.fst.*;
+import org.apache.lucene.search.suggest.fst.FSTCompletionLookup;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.spelling.suggest.LookupFactory;
@@ -27,6 +27,12 @@ import org.apache.solr.spelling.suggest.LookupFactory;
  * Factory for {@link FSTCompletionLookup}
  */
 public class FSTLookupFactory extends LookupFactory {
+  
+  /**
+   * File name for the automaton.
+   */
+  private static final String FILENAME = "fst.bin";
+  
   /**
    * The number of separate buckets for weights (discretization). The more buckets,
    * the more fine-grained term weights (priorities) can be assigned. The speed of lookup
@@ -56,4 +62,9 @@ public class FSTLookupFactory extends LookupFactory {
 
     return new FSTCompletionLookup(buckets, exactMatchFirst);
   }
+
+  @Override
+  public String storeFileName() {
+    return FILENAME;
+  }
 }

solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java

@@ -17,6 +17,8 @@ package org.apache.solr.spelling.suggest.fst;
  * limitations under the License.
  */
 
+import java.io.File;
+
 import org.apache.lucene.search.suggest.Lookup;
 import org.apache.lucene.search.suggest.fst.*;
 import org.apache.solr.common.util.NamedList;
@@ -33,6 +35,12 @@ public class WFSTLookupFactory extends LookupFactory {
    * of other strings in the automaton (possibly with larger weights). 
    */
   public static final String EXACT_MATCH_FIRST = "exactMatchFirst";
+  
+  /**
+   * File name for the automaton.
+   * 
+   */
+  private static final String FILENAME = "wfst.bin";
 
   @Override
   public Lookup create(NamedList params, SolrCore core) {
@@ -42,4 +50,9 @@ public class WFSTLookupFactory extends LookupFactory {
 
     return new WFSTCompletionLookup(exactMatchFirst);
   }
+
+  @Override
+  public String storeFileName() {
+    return FILENAME;
+  }
 }

solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java

@@ -30,10 +30,16 @@ import org.slf4j.LoggerFactory;
  */
 public class JaspellLookupFactory extends LookupFactory {
   private static final Logger LOG = LoggerFactory.getLogger(JaspellLookup.class);
+  private static final String FILENAME = "jaspell.dat";
 
   @Override
   public Lookup create(NamedList params, SolrCore core) {
     LOG.info("init: " + params);
     return new JaspellLookup();
   }
+
+  @Override
+  public String storeFileName() {
+    return FILENAME;
+  }
 }

solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java

@@ -27,9 +27,15 @@ import org.apache.solr.spelling.suggest.LookupFactory;
  * Factory for {@link TSTLookup}
  */
 public class TSTLookupFactory extends LookupFactory {
+  private static final String FILENAME = "tst.dat";
 
   @Override
   public Lookup create(NamedList params, SolrCore core) {
     return new TSTLookup();
   }
+
+  @Override
+  public String storeFileName() {
+    return FILENAME;
+  }
 }