From 49e3866590fdb72661be6ecbec3607e83abc1032 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 7 Mar 2012 19:30:29 +0000 Subject: [PATCH] LUCENE-3807: remove file based store/load from Lookup git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1298049 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/search/suggest/Lookup.java | 17 ------- .../suggest/fst/FSTCompletionLookup.java | 45 +------------------ .../suggest/fst/WFSTCompletionLookup.java | 23 ++-------- .../search/suggest/jaspell/JaspellLookup.java | 23 ---------- .../lucene/search/suggest/tst/TSTLookup.java | 20 --------- .../search/suggest/PersistenceTest.java | 6 ++- .../solr/spelling/suggest/LookupFactory.java | 1 + .../solr/spelling/suggest/Suggester.java | 24 ++++++++-- .../suggest/fst/FSTLookupFactory.java | 13 +++++- .../suggest/fst/WFSTLookupFactory.java | 13 ++++++ .../suggest/jaspell/JaspellLookupFactory.java | 6 +++ .../suggest/tst/TSTLookupFactory.java | 6 +++ 12 files changed, 67 insertions(+), 130 deletions(-) diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java index f6abab61e2f..e200225b2c7 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java @@ -150,21 +150,4 @@ public abstract class Lookup { */ public abstract boolean load(InputStream input) throws IOException; - /** - * Persist the constructed lookup data to a directory. Optional operation. - * @param storeDir directory where data can be stored. - * @return true if successful, false if unsuccessful or not supported. - * @throws IOException when fatal IO error occurs. - */ - public abstract boolean store(File storeDir) throws IOException; - - /** - * Discard current lookup data and load it from a previously saved copy. - * Optional operation. - * @param storeDir directory where lookup data was stored. - * @return true if completed successfully, false if unsuccessful or not supported. - * @throws IOException when fatal IO error occurs. - */ - public abstract boolean load(File storeDir) throws IOException; - } diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java index 9bd0ce79170..dd0b58aa3ea 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java @@ -79,14 +79,6 @@ public class FSTCompletionLookup extends Lookup { */ private final static int sharedTailLength = 5; - /** - * File name for the automaton. - * - * @see #store(File) - * @see #load(File) - */ - private static final String FILENAME = "fst.bin"; - private int buckets; private boolean exactMatchFirst; @@ -264,46 +256,13 @@ public class FSTCompletionLookup extends Lookup { return bucket == -1 ? null : Long.valueOf(bucket); } - /** - * Deserialization from disk. - */ - @Override - public synchronized boolean load(File storeDir) throws IOException { - File data = new File(storeDir, FILENAME); - if (!data.exists() || !data.canRead()) { - return false; - } - - this.higherWeightsCompletion = new FSTCompletion( - FST.read(data, NoOutputs.getSingleton())); - this.normalCompletion = new FSTCompletion( - higherWeightsCompletion.getFST(), false, exactMatchFirst); - - return true; - } - - /** - * Serialization to disk. - */ - @Override - public synchronized boolean store(File storeDir) throws IOException { - if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) { - return false; - } - - if (this.normalCompletion == null) - return false; - - normalCompletion.getFST().save(new File(storeDir, FILENAME)); - return true; - } @Override public synchronized boolean store(OutputStream output) throws IOException { - if (this.normalCompletion == null) - return false; try { + if (this.normalCompletion == null || normalCompletion.getFST() == null) + return false; normalCompletion.getFST().save(new OutputStreamDataOutput(output)); } finally { IOUtils.close(output); diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java index 330cf3c82a4..a373f92006d 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java @@ -17,7 +17,6 @@ package org.apache.lucene.search.suggest.fst; * limitations under the License. */ -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -63,14 +62,6 @@ import org.apache.lucene.util.fst.Util.MinResult; */ public class WFSTCompletionLookup extends Lookup { - /** - * File name for the automaton. - * - * @see #store(File) - * @see #load(File) - */ - private static final String FILENAME = "wfst.bin"; - /** * FST, weights are encoded as costs: (Integer.MAX_VALUE-weight) */ @@ -127,21 +118,13 @@ public class WFSTCompletionLookup extends Lookup { fst = builder.finish(); } - @Override - public boolean store(File storeDir) throws IOException { - fst.save(new File(storeDir, FILENAME)); - return true; - } - - @Override - public boolean load(File storeDir) throws IOException { - this.fst = FST.read(new File(storeDir, FILENAME), PositiveIntOutputs.getSingleton(true)); - return true; - } @Override public boolean store(OutputStream output) throws IOException { try { + if (fst == null) { + return false; + } fst.save(new OutputStreamDataOutput(output)); } finally { IOUtils.close(output); diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java index b7bb15e8a46..9e570671e17 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java @@ -19,9 +19,6 @@ package org.apache.lucene.search.suggest.jaspell; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -109,22 +106,11 @@ public class JaspellLookup extends Lookup { return res; } - public static final String FILENAME = "jaspell.dat"; private static final byte LO_KID = 0x01; private static final byte EQ_KID = 0x02; private static final byte HI_KID = 0x04; private static final byte HAS_VALUE = 0x08; - - @Override - public boolean load(File storeDir) throws IOException { - File data = new File(storeDir, FILENAME); - if (!data.exists() || !data.canRead()) { - return false; - } - return load(new FileInputStream(data)); - } - private void readRecursively(DataInputStream in, TSTNode node) throws IOException { node.splitchar = in.readChar(); byte mask = in.readByte(); @@ -148,15 +134,6 @@ public class JaspellLookup extends Lookup { } } - @Override - public boolean store(File storeDir) throws IOException { - if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) { - return false; - } - File data = new File(storeDir, FILENAME); - return store(new FileOutputStream(data)); - } - private void writeRecursively(DataOutputStream out, TSTNode node) throws IOException { if (node == null) { return; diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java index 99e4e6a8c46..86a10cde713 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java @@ -119,23 +119,12 @@ public class TSTLookup extends Lookup { return res; } - public static final String FILENAME = "tst.dat"; - private static final byte LO_KID = 0x01; private static final byte EQ_KID = 0x02; private static final byte HI_KID = 0x04; private static final byte HAS_TOKEN = 0x08; private static final byte HAS_VALUE = 0x10; - @Override - public synchronized boolean load(File storeDir) throws IOException { - File data = new File(storeDir, FILENAME); - if (!data.exists() || !data.canRead()) { - return false; - } - return load(new FileInputStream(data)); - } - // pre-order traversal private void readRecursively(DataInputStream in, TernaryTreeNode node) throws IOException { node.splitchar = in.readChar(); @@ -160,15 +149,6 @@ public class TSTLookup extends Lookup { } } - @Override - public synchronized boolean store(File storeDir) throws IOException { - if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) { - return false; - } - File data = new File(storeDir, FILENAME); - return store(new FileOutputStream(data)); - } - // pre-order traversal private void writeRecursively(DataOutputStream out, TernaryTreeNode node) throws IOException { // write out the current node diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java index 73f5ae82dad..34bf6b1ed7a 100644 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java +++ b/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java @@ -17,6 +17,8 @@ package org.apache.lucene.search.suggest; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.util.List; import org.apache.lucene.search.suggest.Lookup; @@ -69,11 +71,11 @@ public class PersistenceTest extends LuceneTestCase { // Store the suggester. File storeDir = TEMP_DIR; - lookup.store(storeDir); + lookup.store(new FileOutputStream(new File(storeDir, "lookup.dat"))); // Re-read it from disk. lookup = lookupClass.newInstance(); - lookup.load(storeDir); + lookup.load(new FileInputStream(new File(storeDir, "lookup.dat"))); // Assert validity. long previous = Long.MIN_VALUE; diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java index cd24fa7fb4d..d6212959883 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java @@ -26,4 +26,5 @@ import org.apache.solr.core.SolrCore; */ public abstract class LookupFactory { public abstract Lookup create(NamedList params, SolrCore core); + public abstract String storeFileName(); } diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java index 525ce3b97dc..5286dd9d451 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java @@ -18,6 +18,8 @@ package org.apache.solr.spelling.suggest; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; @@ -73,6 +75,8 @@ public class Suggester extends SolrSpellChecker { protected Lookup lookup; protected String lookupImpl; protected SolrCore core; + + private LookupFactory factory; @Override public String init(NamedList config, SolrCore core) { @@ -92,7 +96,8 @@ public class Suggester extends SolrSpellChecker { lookupImpl = FSTLookupFactory.class.getName(); } - LookupFactory factory = (LookupFactory) core.getResourceLoader().newInstance(lookupImpl); + factory = (LookupFactory) core.getResourceLoader().newInstance(lookupImpl); + lookup = factory.create(config, core); String store = (String)config.get(STORE_DIR); if (store != null) { @@ -105,7 +110,7 @@ public class Suggester extends SolrSpellChecker { } else { // attempt reload of the stored lookup try { - lookup.load(storeDir); + lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName()))); } catch (IOException e) { LOG.warn("Loading stored lookup data failed", e); } @@ -132,8 +137,19 @@ public class Suggester extends SolrSpellChecker { try { lookup.build(dictionary); if (storeDir != null) { - lookup.store(storeDir); + File target = new File(storeDir, factory.storeFileName()); + if(!lookup.store(new FileOutputStream(target))) { + if (sourceLocation == null) { + assert reader != null && field != null; + LOG.error("Store Lookup build from index on field: " + field + " failed reader has: " + reader.maxDoc() + " docs"); + } else { + LOG.error("Store Lookup build from sourceloaction: " + sourceLocation + " failed"); + } + } else { + LOG.info("Stored suggest data to: " + target.getAbsolutePath()); + } } + } catch (Exception e) { LOG.error("Error while building or storing Suggester data", e); } @@ -144,7 +160,7 @@ public class Suggester extends SolrSpellChecker { LOG.info("reload()"); if (dictionary == null && storeDir != null) { // this may be a firstSearcher event, try loading it - if (lookup.load(storeDir)) { + if (lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())))) { return; // loaded ok } LOG.debug("load failed, need to build Lookup again"); diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java index ed7d86e2391..b32af8ee184 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java @@ -18,7 +18,7 @@ package org.apache.solr.spelling.suggest.fst; */ import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.search.suggest.fst.*; +import org.apache.lucene.search.suggest.fst.FSTCompletionLookup; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.spelling.suggest.LookupFactory; @@ -27,6 +27,12 @@ import org.apache.solr.spelling.suggest.LookupFactory; * Factory for {@link FSTCompletionLookup} */ public class FSTLookupFactory extends LookupFactory { + + /** + * File name for the automaton. + */ + private static final String FILENAME = "fst.bin"; + /** * The number of separate buckets for weights (discretization). The more buckets, * the more fine-grained term weights (priorities) can be assigned. The speed of lookup @@ -56,4 +62,9 @@ public class FSTLookupFactory extends LookupFactory { return new FSTCompletionLookup(buckets, exactMatchFirst); } + + @Override + public String storeFileName() { + return FILENAME; + } } diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java index 233fe1e3ada..523816a5d40 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java @@ -17,6 +17,8 @@ package org.apache.solr.spelling.suggest.fst; * limitations under the License. */ +import java.io.File; + import org.apache.lucene.search.suggest.Lookup; import org.apache.lucene.search.suggest.fst.*; import org.apache.solr.common.util.NamedList; @@ -33,6 +35,12 @@ public class WFSTLookupFactory extends LookupFactory { * of other strings in the automaton (possibly with larger weights). */ public static final String EXACT_MATCH_FIRST = "exactMatchFirst"; + + /** + * File name for the automaton. + * + */ + private static final String FILENAME = "wfst.bin"; @Override public Lookup create(NamedList params, SolrCore core) { @@ -42,4 +50,9 @@ public class WFSTLookupFactory extends LookupFactory { return new WFSTCompletionLookup(exactMatchFirst); } + + @Override + public String storeFileName() { + return FILENAME; + } } diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java index 720d9593500..94a7f7a68b5 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java @@ -30,10 +30,16 @@ import org.slf4j.LoggerFactory; */ public class JaspellLookupFactory extends LookupFactory { private static final Logger LOG = LoggerFactory.getLogger(JaspellLookup.class); + private static final String FILENAME = "jaspell.dat"; @Override public Lookup create(NamedList params, SolrCore core) { LOG.info("init: " + params); return new JaspellLookup(); } + + @Override + public String storeFileName() { + return FILENAME; + } } diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java index 37e7e4b7e86..a5aa8ab25d8 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java @@ -27,9 +27,15 @@ import org.apache.solr.spelling.suggest.LookupFactory; * Factory for {@link TSTLookup} */ public class TSTLookupFactory extends LookupFactory { + private static final String FILENAME = "tst.dat"; @Override public Lookup create(NamedList params, SolrCore core) { return new TSTLookup(); } + + @Override + public String storeFileName() { + return FILENAME; + } }