Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3807: remove file based store/load from Lookup 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1298049 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Simon Willnauer 2012-03-07 19:30:29 +00:00
parent 421f8d3efa
commit 49e3866590
12 changed files with 67 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
View File

@ -150,21 +150,4 @@ public abstract class Lookup {
*/
public abstract boolean load(InputStream input) throws IOException;
/**
* Persist the constructed lookup data to a directory. Optional operation.
* @param storeDir directory where data can be stored.
* @return true if successful, false if unsuccessful or not supported.
* @throws IOException when fatal IO error occurs.
*/
public abstract boolean store(File storeDir) throws IOException;
/**
* Discard current lookup data and load it from a previously saved copy.
* Optional operation.
* @param storeDir directory where lookup data was stored.
* @return true if completed successfully, false if unsuccessful or not supported.
* @throws IOException when fatal IO error occurs.
*/
public abstract boolean load(File storeDir) throws IOException;
}

45
modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
View File

@ -79,14 +79,6 @@ public class FSTCompletionLookup extends Lookup {
*/
private final static int sharedTailLength = 5;
/**
* File name for the automaton.
*
* @see #store(File)
* @see #load(File)
*/
private static final String FILENAME = "fst.bin";
private int buckets;
private boolean exactMatchFirst;
@ -264,46 +256,13 @@ public class FSTCompletionLookup extends Lookup {
return bucket == -1 ? null : Long.valueOf(bucket);
}
/**
* Deserialization from disk.
*/
@Override
public synchronized boolean load(File storeDir) throws IOException {
File data = new File(storeDir, FILENAME);
if (!data.exists() || !data.canRead()) {
return false;
}
this.higherWeightsCompletion = new FSTCompletion(
FST.read(data, NoOutputs.getSingleton()));
this.normalCompletion = new FSTCompletion(
higherWeightsCompletion.getFST(), false, exactMatchFirst);
return true;
}
/**
* Serialization to disk.
*/
@Override
public synchronized boolean store(File storeDir) throws IOException {
if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
return false;
}
if (this.normalCompletion == null)
return false;
normalCompletion.getFST().save(new File(storeDir, FILENAME));
return true;
}
@Override
public synchronized boolean store(OutputStream output) throws IOException {
if (this.normalCompletion == null)
return false;
try {
if (this.normalCompletion == null || normalCompletion.getFST() == null)
return false;
normalCompletion.getFST().save(new OutputStreamDataOutput(output));
} finally {
IOUtils.close(output);

23
modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
View File

@ -17,7 +17,6 @@ package org.apache.lucene.search.suggest.fst;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -63,14 +62,6 @@ import org.apache.lucene.util.fst.Util.MinResult;
*/
public class WFSTCompletionLookup extends Lookup {
/**
* File name for the automaton.
*
* @see #store(File)
* @see #load(File)
*/
private static final String FILENAME = "wfst.bin";
/**
* FST<Long>, weights are encoded as costs: (Integer.MAX_VALUE-weight)
*/
@ -127,21 +118,13 @@ public class WFSTCompletionLookup extends Lookup {
fst = builder.finish();
}
@Override
public boolean store(File storeDir) throws IOException {
fst.save(new File(storeDir, FILENAME));
return true;
}
@Override
public boolean load(File storeDir) throws IOException {
this.fst = FST.read(new File(storeDir, FILENAME), PositiveIntOutputs.getSingleton(true));
return true;
}
@Override
public boolean store(OutputStream output) throws IOException {
try {
if (fst == null) {
return false;
}
fst.save(new OutputStreamDataOutput(output));
} finally {
IOUtils.close(output);

23
modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
View File

@ -19,9 +19,6 @@ package org.apache.lucene.search.suggest.jaspell;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -109,22 +106,11 @@ public class JaspellLookup extends Lookup {
return res;
}
public static final String FILENAME = "jaspell.dat";
private static final byte LO_KID = 0x01;
private static final byte EQ_KID = 0x02;
private static final byte HI_KID = 0x04;
private static final byte HAS_VALUE = 0x08;
@Override
public boolean load(File storeDir) throws IOException {
File data = new File(storeDir, FILENAME);
if (!data.exists() || !data.canRead()) {
return false;
}
return load(new FileInputStream(data));
}
private void readRecursively(DataInputStream in, TSTNode node) throws IOException {
node.splitchar = in.readChar();
byte mask = in.readByte();
@ -148,15 +134,6 @@ public class JaspellLookup extends Lookup {
}
}
@Override
public boolean store(File storeDir) throws IOException {
if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
return false;
}
File data = new File(storeDir, FILENAME);
return store(new FileOutputStream(data));
}
private void writeRecursively(DataOutputStream out, TSTNode node) throws IOException {
if (node == null) {
return;

20
modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
View File

@ -119,23 +119,12 @@ public class TSTLookup extends Lookup {
return res;
}
public static final String FILENAME = "tst.dat";
private static final byte LO_KID = 0x01;
private static final byte EQ_KID = 0x02;
private static final byte HI_KID = 0x04;
private static final byte HAS_TOKEN = 0x08;
private static final byte HAS_VALUE = 0x10;
@Override
public synchronized boolean load(File storeDir) throws IOException {
File data = new File(storeDir, FILENAME);
if (!data.exists() || !data.canRead()) {
return false;
}
return load(new FileInputStream(data));
}
// pre-order traversal
private void readRecursively(DataInputStream in, TernaryTreeNode node) throws IOException {
node.splitchar = in.readChar();
@ -160,15 +149,6 @@ public class TSTLookup extends Lookup {
}
}
@Override
public synchronized boolean store(File storeDir) throws IOException {
if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
return false;
}
File data = new File(storeDir, FILENAME);
return store(new FileOutputStream(data));
}
// pre-order traversal
private void writeRecursively(DataOutputStream out, TernaryTreeNode node) throws IOException {
// write out the current node

6
modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
View File

@ -17,6 +17,8 @@
package org.apache.lucene.search.suggest;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.List;
import org.apache.lucene.search.suggest.Lookup;
@ -69,11 +71,11 @@ public class PersistenceTest extends LuceneTestCase {
// Store the suggester.
File storeDir = TEMP_DIR;
lookup.store(storeDir);
lookup.store(new FileOutputStream(new File(storeDir, "lookup.dat")));
// Re-read it from disk.
lookup = lookupClass.newInstance();
lookup.load(storeDir);
lookup.load(new FileInputStream(new File(storeDir, "lookup.dat")));
// Assert validity.
long previous = Long.MIN_VALUE;

1
solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java
View File

@ -26,4 +26,5 @@ import org.apache.solr.core.SolrCore;
*/
public abstract class LookupFactory {
public abstract Lookup create(NamedList params, SolrCore core);
public abstract String storeFileName();
}

24
solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
View File

@ -18,6 +18,8 @@
package org.apache.solr.spelling.suggest;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
@ -73,6 +75,8 @@ public class Suggester extends SolrSpellChecker {
protected Lookup lookup;
protected String lookupImpl;
protected SolrCore core;
private LookupFactory factory;
@Override
public String init(NamedList config, SolrCore core) {
@ -92,7 +96,8 @@ public class Suggester extends SolrSpellChecker {
lookupImpl = FSTLookupFactory.class.getName();
}
LookupFactory factory = (LookupFactory) core.getResourceLoader().newInstance(lookupImpl);
factory = (LookupFactory) core.getResourceLoader().newInstance(lookupImpl);
lookup = factory.create(config, core);
String store = (String)config.get(STORE_DIR);
if (store != null) {
@ -105,7 +110,7 @@ public class Suggester extends SolrSpellChecker {
} else {
// attempt reload of the stored lookup
try {
lookup.load(storeDir);
lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())));
} catch (IOException e) {
LOG.warn("Loading stored lookup data failed", e);
}
@ -132,8 +137,19 @@ public class Suggester extends SolrSpellChecker {
try {
lookup.build(dictionary);
if (storeDir != null) {
lookup.store(storeDir);
File target = new File(storeDir, factory.storeFileName());
if(!lookup.store(new FileOutputStream(target))) {
if (sourceLocation == null) {
assert reader != null && field != null;
LOG.error("Store Lookup build from index on field: " + field + " failed reader has: " + reader.maxDoc() + " docs");
} else {
LOG.error("Store Lookup build from sourceloaction: " + sourceLocation + " failed");
}
} else {
LOG.info("Stored suggest data to: " + target.getAbsolutePath());
}
}
} catch (Exception e) {
LOG.error("Error while building or storing Suggester data", e);
}
@ -144,7 +160,7 @@ public class Suggester extends SolrSpellChecker {
LOG.info("reload()");
if (dictionary == null && storeDir != null) {
// this may be a firstSearcher event, try loading it
if (lookup.load(storeDir)) {
if (lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())))) {
return; // loaded ok
}
LOG.debug("load failed, need to build Lookup again");

13
solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java
View File

@ -18,7 +18,7 @@ package org.apache.solr.spelling.suggest.fst;
*/
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.fst.*;
import org.apache.lucene.search.suggest.fst.FSTCompletionLookup;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.spelling.suggest.LookupFactory;
@ -27,6 +27,12 @@ import org.apache.solr.spelling.suggest.LookupFactory;
* Factory for {@link FSTCompletionLookup}
*/
public class FSTLookupFactory extends LookupFactory {
/**
* File name for the automaton.
*/
private static final String FILENAME = "fst.bin";
/**
* The number of separate buckets for weights (discretization). The more buckets,
* the more fine-grained term weights (priorities) can be assigned. The speed of lookup
@ -56,4 +62,9 @@ public class FSTLookupFactory extends LookupFactory {
return new FSTCompletionLookup(buckets, exactMatchFirst);
}
@Override
public String storeFileName() {
return FILENAME;
}
}

13
solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java
View File

@ -17,6 +17,8 @@ package org.apache.solr.spelling.suggest.fst;
* limitations under the License.
*/
import java.io.File;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.fst.*;
import org.apache.solr.common.util.NamedList;
@ -33,6 +35,12 @@ public class WFSTLookupFactory extends LookupFactory {
* of other strings in the automaton (possibly with larger weights).
*/
public static final String EXACT_MATCH_FIRST = "exactMatchFirst";
/**
* File name for the automaton.
*
*/
private static final String FILENAME = "wfst.bin";
@Override
public Lookup create(NamedList params, SolrCore core) {
@ -42,4 +50,9 @@ public class WFSTLookupFactory extends LookupFactory {
return new WFSTCompletionLookup(exactMatchFirst);
}
@Override
public String storeFileName() {
return FILENAME;
}
}

6
solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java
View File

@ -30,10 +30,16 @@ import org.slf4j.LoggerFactory;
*/
public class JaspellLookupFactory extends LookupFactory {
private static final Logger LOG = LoggerFactory.getLogger(JaspellLookup.class);
private static final String FILENAME = "jaspell.dat";
@Override
public Lookup create(NamedList params, SolrCore core) {
LOG.info("init: " + params);
return new JaspellLookup();
}
@Override
public String storeFileName() {
return FILENAME;
}
}

6
solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java
View File

@ -27,9 +27,15 @@ import org.apache.solr.spelling.suggest.LookupFactory;
* Factory for {@link TSTLookup}
*/
public class TSTLookupFactory extends LookupFactory {
private static final String FILENAME = "tst.dat";
@Override
public Lookup create(NamedList params, SolrCore core) {
return new TSTLookup();
}
@Override
public String storeFileName() {
return FILENAME;
}
}