mirror of
https://github.com/apache/lucene.git
synced 2025-02-08 19:15:06 +00:00
SOLR-6845: Add a ''buildOnStartup'' option for suggesters. (Tomás Fernández Löbbe)
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1653410 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
e7045badaa
commit
6ce841759a
@ -333,6 +333,8 @@ New Features
|
|||||||
|
|
||||||
* SOLR-5147: Support child documents in DIH
|
* SOLR-5147: Support child documents in DIH
|
||||||
(Vadim Kirilchuk, Shawn Heisey, Thomas Champagne, Mikhail Khludnev via Noble Paul)
|
(Vadim Kirilchuk, Shawn Heisey, Thomas Champagne, Mikhail Khludnev via Noble Paul)
|
||||||
|
|
||||||
|
* SOLR-6845: Add a “buildOnStartup” option for suggesters. (Tomás Fernández Löbbe)
|
||||||
|
|
||||||
Bug Fixes
|
Bug Fixes
|
||||||
----------------------
|
----------------------
|
||||||
|
@ -17,6 +17,7 @@ package org.apache.solr.handler.component;
|
|||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
@ -29,6 +30,7 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
import org.apache.lucene.search.suggest.Lookup;
|
import org.apache.lucene.search.suggest.Lookup;
|
||||||
import org.apache.lucene.search.suggest.Lookup.LookupResult;
|
import org.apache.lucene.search.suggest.Lookup.LookupResult;
|
||||||
@ -77,6 +79,9 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
|
|||||||
/** SolrConfig label to identify boolean value to build suggesters on optimize */
|
/** SolrConfig label to identify boolean value to build suggesters on optimize */
|
||||||
private static final String BUILD_ON_OPTIMIZE_LABEL = "buildOnOptimize";
|
private static final String BUILD_ON_OPTIMIZE_LABEL = "buildOnOptimize";
|
||||||
|
|
||||||
|
/** SolrConfig label to identify boolean value to build suggesters on optimize */
|
||||||
|
private static final String BUILD_ON_STARTUP_LABEL = "buildOnStartup";
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
protected NamedList initParams;
|
protected NamedList initParams;
|
||||||
|
|
||||||
@ -128,14 +133,25 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
|
|||||||
throw new RuntimeException("More than one dictionary is missing name.");
|
throw new RuntimeException("More than one dictionary is missing name.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
boolean buildOnStartup;
|
||||||
// Register event listeners for this Suggester
|
Object buildOnStartupObj = suggesterParams.get(BUILD_ON_STARTUP_LABEL);
|
||||||
core.registerFirstSearcherListener(new SuggesterListener(core, suggester, false, false));
|
if (buildOnStartupObj == null) {
|
||||||
|
File storeFile = suggester.getStoreFile();
|
||||||
|
buildOnStartup = storeFile == null || !storeFile.exists();
|
||||||
|
} else {
|
||||||
|
buildOnStartup = Boolean.parseBoolean((String) buildOnStartupObj);
|
||||||
|
}
|
||||||
boolean buildOnCommit = Boolean.parseBoolean((String) suggesterParams.get(BUILD_ON_COMMIT_LABEL));
|
boolean buildOnCommit = Boolean.parseBoolean((String) suggesterParams.get(BUILD_ON_COMMIT_LABEL));
|
||||||
boolean buildOnOptimize = Boolean.parseBoolean((String) suggesterParams.get(BUILD_ON_OPTIMIZE_LABEL));
|
boolean buildOnOptimize = Boolean.parseBoolean((String) suggesterParams.get(BUILD_ON_OPTIMIZE_LABEL));
|
||||||
if (buildOnCommit || buildOnOptimize) {
|
|
||||||
|
if (!core.isReloaded()) {
|
||||||
|
// Register first searcher event listeners for this Suggester unless it's a core reload
|
||||||
|
core.registerFirstSearcherListener(new SuggesterListener(core, suggester, false, false, buildOnStartup, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (buildOnCommit || buildOnOptimize || core.isReloaded()) {
|
||||||
LOG.info("Registering newSearcher listener for suggester: " + suggester.getName());
|
LOG.info("Registering newSearcher listener for suggester: " + suggester.getName());
|
||||||
core.registerNewSearcherListener(new SuggesterListener(core, suggester, buildOnCommit, buildOnOptimize));
|
core.registerNewSearcherListener(new SuggesterListener(core, suggester, buildOnCommit, buildOnOptimize, buildOnStartup, core.isReloaded()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -448,12 +464,22 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
|
|||||||
private final SolrSuggester suggester;
|
private final SolrSuggester suggester;
|
||||||
private final boolean buildOnCommit;
|
private final boolean buildOnCommit;
|
||||||
private final boolean buildOnOptimize;
|
private final boolean buildOnOptimize;
|
||||||
|
private final boolean buildOnStartup;
|
||||||
|
|
||||||
|
// On core reload, immediately after the core is created a new searcher is opened, causing the suggester
|
||||||
|
// to trigger a "buildOnCommit". The only event that we want to trigger in that situation is "buildOnStartup"
|
||||||
|
// so if buildOnCommit is true and this is a core being reloaded, we will skip the first time this listener
|
||||||
|
// is called
|
||||||
|
private final AtomicBoolean firstCall = new AtomicBoolean(true);
|
||||||
|
private final boolean isCoreReload;
|
||||||
|
|
||||||
public SuggesterListener(SolrCore core, SolrSuggester checker, boolean buildOnCommit, boolean buildOnOptimize) {
|
public SuggesterListener(SolrCore core, SolrSuggester checker, boolean buildOnCommit, boolean buildOnOptimize, boolean buildOnStartup, boolean isCoreReload) {
|
||||||
this.core = core;
|
this.core = core;
|
||||||
this.suggester = checker;
|
this.suggester = checker;
|
||||||
this.buildOnCommit = buildOnCommit;
|
this.buildOnCommit = buildOnCommit;
|
||||||
this.buildOnOptimize = buildOnOptimize;
|
this.buildOnOptimize = buildOnOptimize;
|
||||||
|
this.buildOnStartup = buildOnStartup;
|
||||||
|
this.isCoreReload = isCoreReload;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -462,13 +488,12 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
|
|||||||
@Override
|
@Override
|
||||||
public void newSearcher(SolrIndexSearcher newSearcher,
|
public void newSearcher(SolrIndexSearcher newSearcher,
|
||||||
SolrIndexSearcher currentSearcher) {
|
SolrIndexSearcher currentSearcher) {
|
||||||
if (currentSearcher == null) {
|
assert !(currentSearcher == null && isCoreReload):
|
||||||
// firstSearcher event
|
"SuggesterListener should not be added as firstSearcherListener in case of a core reload";
|
||||||
try {
|
boolean first = firstCall.getAndSet(false);
|
||||||
LOG.info("Loading suggester index for: " + suggester.getName());
|
if (currentSearcher == null || (isCoreReload && first)) {
|
||||||
suggester.reload(core, newSearcher);
|
if (buildOnStartup) {
|
||||||
} catch (IOException e) {
|
buildSuggesterIndex(newSearcher);
|
||||||
log.error("Exception in reloading suggester index for: " + suggester.getName(), e);
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// newSearcher event
|
// newSearcher event
|
||||||
|
@ -23,7 +23,6 @@ import java.io.FileInputStream;
|
|||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.lucene.search.spell.Dictionary;
|
import org.apache.lucene.search.spell.Dictionary;
|
||||||
@ -133,7 +132,7 @@ public class SolrSuggester implements Accountable {
|
|||||||
} else {
|
} else {
|
||||||
// attempt reload of the stored lookup
|
// attempt reload of the stored lookup
|
||||||
try {
|
try {
|
||||||
lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())));
|
lookup.load(new FileInputStream(getStoreFile()));
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.warn("Loading stored lookup data failed, possibly not cached yet");
|
LOG.warn("Loading stored lookup data failed, possibly not cached yet");
|
||||||
}
|
}
|
||||||
@ -161,7 +160,7 @@ public class SolrSuggester implements Accountable {
|
|||||||
dictionary = dictionaryFactory.create(core, searcher);
|
dictionary = dictionaryFactory.create(core, searcher);
|
||||||
lookup.build(dictionary);
|
lookup.build(dictionary);
|
||||||
if (storeDir != null) {
|
if (storeDir != null) {
|
||||||
File target = new File(storeDir, factory.storeFileName());
|
File target = getStoreFile();
|
||||||
if(!lookup.store(new FileOutputStream(target))) {
|
if(!lookup.store(new FileOutputStream(target))) {
|
||||||
LOG.error("Store Lookup build failed");
|
LOG.error("Store Lookup build failed");
|
||||||
} else {
|
} else {
|
||||||
@ -174,19 +173,33 @@ public class SolrSuggester implements Accountable {
|
|||||||
public void reload(SolrCore core, SolrIndexSearcher searcher) throws IOException {
|
public void reload(SolrCore core, SolrIndexSearcher searcher) throws IOException {
|
||||||
LOG.info("reload()");
|
LOG.info("reload()");
|
||||||
if (dictionary == null && storeDir != null) {
|
if (dictionary == null && storeDir != null) {
|
||||||
// this may be a firstSearcher event, try loading it
|
File lookupFile = getStoreFile();
|
||||||
FileInputStream is = new FileInputStream(new File(storeDir, factory.storeFileName()));
|
if (lookupFile.exists()) {
|
||||||
try {
|
// this may be a firstSearcher event, try loading it
|
||||||
if (lookup.load(is)) {
|
FileInputStream is = new FileInputStream(lookupFile);
|
||||||
return; // loaded ok
|
try {
|
||||||
|
if (lookup.load(is)) {
|
||||||
|
return; // loaded ok
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
IOUtils.closeWhileHandlingException(is);
|
||||||
}
|
}
|
||||||
} finally {
|
} else {
|
||||||
IOUtils.closeWhileHandlingException(is);
|
LOG.info("lookup file doesn't exist");
|
||||||
}
|
}
|
||||||
LOG.debug("load failed, need to build Lookup again");
|
|
||||||
}
|
}
|
||||||
// loading was unsuccessful - build it again
|
}
|
||||||
build(core, searcher);
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @return the file where this suggester is stored.
|
||||||
|
* null if no storeDir was configured
|
||||||
|
*/
|
||||||
|
public File getStoreFile() {
|
||||||
|
if (storeDir == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new File(storeDir, factory.storeFileName());
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns suggestions based on the {@link SuggesterOptions} passed */
|
/** Returns suggestions based on the {@link SuggesterOptions} passed */
|
||||||
|
@ -23,7 +23,7 @@
|
|||||||
solr.StandardDirectoryFactory, the default, is filesystem based.
|
solr.StandardDirectoryFactory, the default, is filesystem based.
|
||||||
solr.RAMDirectoryFactory is memory based and not persistent. -->
|
solr.RAMDirectoryFactory is memory based and not persistent. -->
|
||||||
<dataDir>${solr.data.dir:}</dataDir>
|
<dataDir>${solr.data.dir:}</dataDir>
|
||||||
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
|
<directoryFactory name="DirectoryFactory" class="solr.NRTCachingDirectoryFactory"/>
|
||||||
|
|
||||||
<updateHandler class="solr.DirectUpdateHandler2"/>
|
<updateHandler class="solr.DirectUpdateHandler2"/>
|
||||||
|
|
||||||
@ -40,7 +40,6 @@
|
|||||||
<str name="suggestAnalyzerFieldType">text</str>
|
<str name="suggestAnalyzerFieldType">text</str>
|
||||||
<str name="buildOnCommit">true</str>
|
<str name="buildOnCommit">true</str>
|
||||||
|
|
||||||
<!-- Suggester properties -->
|
|
||||||
<float name="threshold">0.0</float>
|
<float name="threshold">0.0</float>
|
||||||
</lst>
|
</lst>
|
||||||
|
|
||||||
@ -61,9 +60,9 @@
|
|||||||
<str name="dictionaryImpl">DocumentDictionaryFactory</str>
|
<str name="dictionaryImpl">DocumentDictionaryFactory</str>
|
||||||
<str name="field">cat</str>
|
<str name="field">cat</str>
|
||||||
<str name="weightField">price</str>
|
<str name="weightField">price</str>
|
||||||
<str name="storeDir">suggest_fuzzy_doc_dict_payload</str>
|
|
||||||
<str name="suggestAnalyzerFieldType">text</str>
|
<str name="suggestAnalyzerFieldType">text</str>
|
||||||
<str name="buildOnCommit">true</str>
|
<str name="buildOnCommit">true</str>
|
||||||
|
<str name="buildOnStartup">false</str>
|
||||||
</lst>
|
</lst>
|
||||||
|
|
||||||
<!-- Suggest component (Document Expression Dictionary) -->
|
<!-- Suggest component (Document Expression Dictionary) -->
|
||||||
@ -79,6 +78,57 @@
|
|||||||
<str name="suggestAnalyzerFieldType">text</str>
|
<str name="suggestAnalyzerFieldType">text</str>
|
||||||
<str name="buildOnCommit">true</str>
|
<str name="buildOnCommit">true</str>
|
||||||
</lst>
|
</lst>
|
||||||
|
|
||||||
|
<!-- Suggest component (Document Dictionary) that is built on startup-->
|
||||||
|
<lst name="suggester">
|
||||||
|
<str name="name">suggest_fuzzy_doc_dict_build_startup</str>
|
||||||
|
<str name="lookupImpl">FuzzyLookupFactory</str>
|
||||||
|
<str name="dictionaryImpl">DocumentDictionaryFactory</str>
|
||||||
|
<str name="field">cat</str>
|
||||||
|
<str name="weightField">price</str>
|
||||||
|
<str name="suggestAnalyzerFieldType">text</str>
|
||||||
|
<str name="buildOnCommit">false</str>
|
||||||
|
<str name="buildOnStartup">true</str>
|
||||||
|
</lst>
|
||||||
|
|
||||||
|
<!-- Suggest component (Document Dictionary) only builds manually-->
|
||||||
|
<lst name="suggester">
|
||||||
|
<str name="name">suggest_fuzzy_doc_manal_build</str>
|
||||||
|
<str name="lookupImpl">FuzzyLookupFactory</str>
|
||||||
|
<str name="dictionaryImpl">DocumentDictionaryFactory</str>
|
||||||
|
<str name="field">cat</str>
|
||||||
|
<str name="weightField">price</str>
|
||||||
|
<str name="suggestAnalyzerFieldType">text</str>
|
||||||
|
<str name="buildOnCommit">false</str>
|
||||||
|
<str name="buildOnStartup">false</str>
|
||||||
|
<str name="storeDir">suggest_fuzzy_doc_manal_build</str>
|
||||||
|
</lst>
|
||||||
|
|
||||||
|
<!-- Suggest component (Document Dictionary) only builds manually and
|
||||||
|
has the default buildOnStartup behavior -->
|
||||||
|
<lst name="suggester">
|
||||||
|
<str name="name">suggest_doc_default_startup</str>
|
||||||
|
<str name="lookupImpl">AnalyzingLookupFactory</str>
|
||||||
|
<str name="dictionaryImpl">DocumentDictionaryFactory</str>
|
||||||
|
<str name="field">cat</str>
|
||||||
|
<str name="weightField">price</str>
|
||||||
|
<str name="suggestAnalyzerFieldType">text</str>
|
||||||
|
<str name="buildOnCommit">false</str>
|
||||||
|
<str name="storeDir">suggest_doc_default_startup</str>
|
||||||
|
</lst>
|
||||||
|
|
||||||
|
<!-- Suggest component (Document Dictionary) only builds manually and
|
||||||
|
has the default buildOnStartup behavior with no storeDir -->
|
||||||
|
<lst name="suggester">
|
||||||
|
<str name="name">suggest_doc_default_startup_no_store</str>
|
||||||
|
<str name="lookupImpl">AnalyzingLookupFactory</str>
|
||||||
|
<str name="dictionaryImpl">DocumentDictionaryFactory</str>
|
||||||
|
<str name="field">cat</str>
|
||||||
|
<str name="weightField">price</str>
|
||||||
|
<str name="suggestAnalyzerFieldType">text</str>
|
||||||
|
<str name="buildOnCommit">false</str>
|
||||||
|
</lst>
|
||||||
|
|
||||||
</searchComponent>
|
</searchComponent>
|
||||||
|
|
||||||
<requestHandler name="/suggest" class="org.apache.solr.handler.component.SearchHandler">
|
<requestHandler name="/suggest" class="org.apache.solr.handler.component.SearchHandler">
|
||||||
@ -91,5 +141,7 @@
|
|||||||
</requestHandler>
|
</requestHandler>
|
||||||
|
|
||||||
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
|
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
|
||||||
|
|
||||||
|
<query><useColdSearcher>false</useColdSearcher></query>
|
||||||
|
|
||||||
</config>
|
</config>
|
||||||
|
@ -19,7 +19,12 @@ package org.apache.solr.handler.component;
|
|||||||
|
|
||||||
import org.apache.solr.SolrTestCaseJ4;
|
import org.apache.solr.SolrTestCaseJ4;
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
|
import org.apache.solr.core.CoreContainer;
|
||||||
|
import org.apache.solr.core.CoreDescriptor;
|
||||||
|
import org.apache.solr.core.SolrCore;
|
||||||
|
import org.apache.solr.search.SolrIndexSearcher;
|
||||||
import org.apache.solr.spelling.suggest.SuggesterParams;
|
import org.apache.solr.spelling.suggest.SuggesterParams;
|
||||||
|
import org.apache.solr.util.RefCounted;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
@ -58,6 +63,11 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
|
|||||||
assertU(delQ("*:*"));
|
assertU(delQ("*:*"));
|
||||||
optimize();
|
optimize();
|
||||||
assertU((commit()));
|
assertU((commit()));
|
||||||
|
// rebuild suggesters with empty index
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_BUILD_ALL, "true"),
|
||||||
|
"//str[@name='command'][.='buildAll']"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -187,4 +197,288 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDefaultBuildOnStartupNotStoredDict() throws Exception {
|
||||||
|
|
||||||
|
final String suggester = "suggest_doc_default_startup_no_store";
|
||||||
|
|
||||||
|
// validate that this suggester is not storing the lookup
|
||||||
|
assertEquals(suggester,
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='name']", false));
|
||||||
|
assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='storeDir']", false));
|
||||||
|
|
||||||
|
// validate that this suggester only builds manually and has not buildOnStartup parameter
|
||||||
|
assertEquals("false",
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='buildOnCommit']", true));
|
||||||
|
assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='buildOnStartup']", false));
|
||||||
|
|
||||||
|
reloadCore(random().nextBoolean());
|
||||||
|
|
||||||
|
// Validate that the suggester was built on new/reload core
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "example",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
// add one more doc, this should not be seen after a core reload (not until the suggester is manually rebuilt)
|
||||||
|
assertU(adoc("id", "10", "cat", "example data extra ", "price", "40", "weight", "35"));
|
||||||
|
assertU((commit()));
|
||||||
|
|
||||||
|
reloadCore(random().nextBoolean());
|
||||||
|
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "example",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "3"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='3']"
|
||||||
|
);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDefaultBuildOnStartupStoredDict() throws Exception {
|
||||||
|
|
||||||
|
final String suggester = "suggest_doc_default_startup";
|
||||||
|
|
||||||
|
// validate that this suggester is storing the lookup
|
||||||
|
assertEquals(suggester,
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='name']", false));
|
||||||
|
assertEquals(suggester,
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='storeDir']", false));
|
||||||
|
|
||||||
|
// validate that this suggester only builds manually and has not buildOnStartup parameter
|
||||||
|
assertEquals("false",
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='buildOnCommit']", true));
|
||||||
|
assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='buildOnStartup']", false));
|
||||||
|
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "example",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='0']"
|
||||||
|
);
|
||||||
|
|
||||||
|
// build the suggester manually
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_BUILD, "true"),
|
||||||
|
"//str[@name='command'][.='build']"
|
||||||
|
);
|
||||||
|
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "example",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
reloadCore(random().nextBoolean());
|
||||||
|
|
||||||
|
// Validate that the suggester was loaded on new/reload core
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "example",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
// add one more doc, this should not be seen after a core reload (not until the suggester is manually rebuilt)
|
||||||
|
assertU(adoc("id", "10", "cat", "example data extra ", "price", "40", "weight", "35"));
|
||||||
|
assertU((commit()));
|
||||||
|
|
||||||
|
reloadCore(random().nextBoolean());
|
||||||
|
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "example",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "3"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
// build the suggester manually
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_BUILD, "true"),
|
||||||
|
"//str[@name='command'][.='build']"
|
||||||
|
);
|
||||||
|
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "example",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "3"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='3']"
|
||||||
|
);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLoadOnStartup() throws Exception {
|
||||||
|
|
||||||
|
final String suggester = "suggest_fuzzy_doc_manal_build";
|
||||||
|
|
||||||
|
// validate that this suggester is storing the lookup
|
||||||
|
assertEquals(suggester,
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='name']", false));
|
||||||
|
assertEquals(suggester,
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='storeDir']", false));
|
||||||
|
|
||||||
|
// validate that this suggester only builds manually
|
||||||
|
assertEquals("false",
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='buildOnCommit']", true));
|
||||||
|
assertEquals("false",
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='buildOnStartup']", true));
|
||||||
|
|
||||||
|
// build the suggester manually
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_BUILD, "true"),
|
||||||
|
"//str[@name='command'][.='build']"
|
||||||
|
);
|
||||||
|
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "exampel",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
reloadCore(false);
|
||||||
|
|
||||||
|
// Validate that the suggester was loaded on core reload
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "exampel",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
reloadCore(true);
|
||||||
|
|
||||||
|
// Validate that the suggester was loaded on new core
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggester,
|
||||||
|
SuggesterParams.SUGGEST_Q, "exampel",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testBuildOnStartupWithCoreReload() throws Exception {
|
||||||
|
doTestBuildOnStartup(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testBuildOnStartupWithNewCores() throws Exception {
|
||||||
|
doTestBuildOnStartup(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doTestBuildOnStartup(boolean createNewCores) throws Exception {
|
||||||
|
|
||||||
|
final String suggesterFuzzy = "suggest_fuzzy_doc_dict";
|
||||||
|
|
||||||
|
// the test relies on useColdSearcher=false
|
||||||
|
assertFalse("Precondition not met for test. useColdSearcher must be false",
|
||||||
|
h.getCore().getSolrConfig().useColdSearcher);
|
||||||
|
|
||||||
|
// validate that this suggester is not storing the lookup and buildOnStartup is not set
|
||||||
|
assertEquals(suggesterFuzzy,
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='name']", false));
|
||||||
|
assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='storeDir']", false));
|
||||||
|
|
||||||
|
// assert that buildOnStartup=false
|
||||||
|
assertEquals("false",
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='buildOnStartup']", false));
|
||||||
|
|
||||||
|
// verify that this suggester is built (there was a commit in setUp)
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggesterFuzzy,
|
||||||
|
SuggesterParams.SUGGEST_Q, "exampel",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggesterFuzzy + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
// reload the core and wait for for the listeners to finish
|
||||||
|
reloadCore(createNewCores);
|
||||||
|
if (System.getProperty(SYSPROP_NIGHTLY) != null) {
|
||||||
|
// wait some time here in nightly to make sure there are no race conditions in suggester build
|
||||||
|
Thread.sleep(1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The suggester should be empty
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggesterFuzzy,
|
||||||
|
SuggesterParams.SUGGEST_Q, "exampel",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggesterFuzzy + "']/lst[@name='exampel']/int[@name='numFound'][.='0']"
|
||||||
|
);
|
||||||
|
|
||||||
|
// build the suggester manually
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggesterFuzzy,
|
||||||
|
SuggesterParams.SUGGEST_BUILD, "true"),
|
||||||
|
"//str[@name='command'][.='build']"
|
||||||
|
);
|
||||||
|
|
||||||
|
// validate the suggester is built again
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggesterFuzzy,
|
||||||
|
SuggesterParams.SUGGEST_Q, "exampel",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggesterFuzzy + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
|
||||||
|
final String suggestStartup = "suggest_fuzzy_doc_dict_build_startup";
|
||||||
|
|
||||||
|
// repeat the test with "suggest_fuzzy_doc_dict_build_startup", it is exactly the same but with buildOnStartup=true
|
||||||
|
assertEquals(suggestStartup,
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='name']", false));
|
||||||
|
assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='storeDir']", false));
|
||||||
|
assertEquals("true",
|
||||||
|
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='buildOnStartup']", false));
|
||||||
|
|
||||||
|
// reload the core
|
||||||
|
reloadCore(createNewCores);
|
||||||
|
// verify that this suggester is built (should build on startup)
|
||||||
|
assertQ(req("qt", rh,
|
||||||
|
SuggesterParams.SUGGEST_DICT, suggestStartup,
|
||||||
|
SuggesterParams.SUGGEST_Q, "exampel",
|
||||||
|
SuggesterParams.SUGGEST_COUNT, "2"),
|
||||||
|
"//lst[@name='suggest']/lst[@name='" + suggestStartup + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void reloadCore(boolean createNewCore) throws Exception {
|
||||||
|
if (createNewCore) {
|
||||||
|
CoreContainer cores = h.getCoreContainer();
|
||||||
|
SolrCore core = h.getCore();
|
||||||
|
String dataDir1 = core.getDataDir();
|
||||||
|
CoreDescriptor cd = core.getCoreDescriptor();
|
||||||
|
cores.unload(core.getName());
|
||||||
|
SolrCore createdCore = cores.create(cd);
|
||||||
|
assertEquals(dataDir1, createdCore.getDataDir());
|
||||||
|
assertEquals(createdCore, h.getCore());
|
||||||
|
} else {
|
||||||
|
h.reload();
|
||||||
|
// On regular reloading, wait until the new searcher is registered
|
||||||
|
RefCounted<SolrIndexSearcher> registeredSearcher = h.getCore().getRegisteredSearcher();
|
||||||
|
RefCounted<SolrIndexSearcher> newestSearcher = h.getCore().getNewestSearcher(false);;
|
||||||
|
while (registeredSearcher.get() != newestSearcher.get()) {
|
||||||
|
registeredSearcher.decref();
|
||||||
|
newestSearcher.decref();
|
||||||
|
Thread.sleep(50);
|
||||||
|
registeredSearcher = h.getCore().getRegisteredSearcher();
|
||||||
|
newestSearcher = h.getCore().getNewestSearcher(false);
|
||||||
|
}
|
||||||
|
registeredSearcher.decref();
|
||||||
|
newestSearcher.decref();
|
||||||
|
}
|
||||||
|
|
||||||
|
assertQ(req("qt", "standard",
|
||||||
|
"q", "*:*"),
|
||||||
|
"//*[@numFound='11']"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -1339,8 +1339,7 @@
|
|||||||
|
|
||||||
To use this suggester, set the "solr.suggester.enabled=true" system property
|
To use this suggester, set the "solr.suggester.enabled=true" system property
|
||||||
-->
|
-->
|
||||||
<searchComponent name="suggest" class="solr.SuggestComponent"
|
<searchComponent name="suggest" class="solr.SuggestComponent">
|
||||||
enable="${solr.suggester.enabled:false}" >
|
|
||||||
<lst name="suggester">
|
<lst name="suggester">
|
||||||
<str name="name">mySuggester</str>
|
<str name="name">mySuggester</str>
|
||||||
<str name="lookupImpl">FuzzyLookupFactory</str>
|
<str name="lookupImpl">FuzzyLookupFactory</str>
|
||||||
@ -1348,6 +1347,7 @@
|
|||||||
<str name="field">cat</str>
|
<str name="field">cat</str>
|
||||||
<str name="weightField">price</str>
|
<str name="weightField">price</str>
|
||||||
<str name="suggestAnalyzerFieldType">string</str>
|
<str name="suggestAnalyzerFieldType">string</str>
|
||||||
|
<str name="buildOnStartup">false</str>
|
||||||
</lst>
|
</lst>
|
||||||
</searchComponent>
|
</searchComponent>
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user