Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6845: Fixed Suggester's buildOnStartup in core reload and improved tests 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1654710 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tomas Eduardo Fernandez Lobbe 2015-01-25 22:40:44 +00:00
parent 9333716acb
commit 5c809f285c
3 changed files with 107 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
View File

@ -30,7 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
@ -144,14 +144,11 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
boolean buildOnCommit = Boolean.parseBoolean((String) suggesterParams.get(BUILD_ON_COMMIT_LABEL));
boolean buildOnOptimize = Boolean.parseBoolean((String) suggesterParams.get(BUILD_ON_OPTIMIZE_LABEL));
if (!core.isReloaded()) {
// Register first searcher event listeners for this Suggester unless it's a core reload
core.registerFirstSearcherListener(new SuggesterListener(core, suggester, false, false, buildOnStartup, false));
}
if (buildOnCommit || buildOnOptimize || core.isReloaded()) {
LOG.info("Registering newSearcher listener for suggester: " + suggester.getName());
core.registerNewSearcherListener(new SuggesterListener(core, suggester, buildOnCommit, buildOnOptimize, buildOnStartup, core.isReloaded()));
if (buildOnCommit || buildOnOptimize || buildOnStartup) {
SuggesterListener listener = new SuggesterListener(core, suggester, buildOnCommit, buildOnOptimize, buildOnStartup, core.isReloaded());
LOG.info("Registering searcher listener for suggester: " + suggester.getName() + " - " + listener);
core.registerFirstSearcherListener(listener);
core.registerNewSearcherListener(listener);
}
}
}
@ -469,9 +466,10 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
// On core reload, immediately after the core is created a new searcher is opened, causing the suggester
// to trigger a "buildOnCommit". The only event that we want to trigger in that situation is "buildOnStartup"
// so if buildOnCommit is true and this is a core being reloaded, we will skip the first time this listener
// is called
private final AtomicBoolean firstCall = new AtomicBoolean(true);
// is called.
private final AtomicLong callCount = new AtomicLong(0);
private final boolean isCoreReload;
public SuggesterListener(SolrCore core, SolrSuggester checker, boolean buildOnCommit, boolean buildOnOptimize, boolean buildOnStartup, boolean isCoreReload) {
this.core = core;
@ -488,23 +486,23 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
@Override
public void newSearcher(SolrIndexSearcher newSearcher,
SolrIndexSearcher currentSearcher) {
assert !(currentSearcher == null && isCoreReload):
"SuggesterListener should not be added as firstSearcherListener in case of a core reload";
boolean first = firstCall.getAndSet(false);
if (currentSearcher == null || (isCoreReload && first)) {
long thisCallCount = callCount.incrementAndGet();
if (isCoreReload && thisCallCount == 1) {
LOG.info("Skipping first newSearcher call for suggester " + suggester + " in core reload");
return;
} else if (thisCallCount == 1 || (isCoreReload && thisCallCount == 2)) {
if (buildOnStartup) {
LOG.info("buildOnStartup: " + suggester.getName());
buildSuggesterIndex(newSearcher);
}
} else {
// newSearcher event
if (buildOnCommit) {
LOG.info("buildOnCommit: " + suggester.getName());
buildSuggesterIndex(newSearcher);
} else if (buildOnOptimize) {
if (newSearcher.getIndexReader().leaves().size() == 1) {
LOG.info("buildOnOptimize: " + suggester.getName());
buildSuggesterIndex(newSearcher);
} else {
LOG.info("Index is not optimized therefore skipping building suggester index for: "
+ suggester.getName());
}
}
}
@ -513,7 +511,6 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
private void buildSuggesterIndex(SolrIndexSearcher newSearcher) {
try {
LOG.info("Building suggester index for: " + suggester.getName());
suggester.build(core, newSearcher);
} catch (Exception e) {
log.error("Exception in building suggester index for: " + suggester.getName(), e);
@ -525,6 +522,14 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
@Override
public void postSoftCommit() {}
@Override
public String toString() {
return "SuggesterListener [core=" + core + ", suggester=" + suggester
+ ", buildOnCommit=" + buildOnCommit + ", buildOnOptimize="
+ buildOnOptimize + ", buildOnStartup=" + buildOnStartup
+ ", isCoreReload=" + isCoreReload + "]";
}
}
}

6
solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java
View File

@ -122,7 +122,7 @@ public class SolrSuggester implements Accountable {
});
// if store directory is provided make it or load up the lookup with its content
if (store != null) {
if (store != null && !store.isEmpty()) {
storeDir = new File(store);
if (!storeDir.isAbsolute()) {
storeDir = new File(core.getDataDir() + File.separator + storeDir);
@ -157,7 +157,7 @@ public class SolrSuggester implements Accountable {
/** Build the underlying Lucene Suggester */
public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
LOG.info("build(" + name + ")");
LOG.info("SolrSuggester.build(" + name + ")");
dictionary = dictionaryFactory.create(core, searcher);
lookup.build(dictionary);
@ -173,7 +173,7 @@ public class SolrSuggester implements Accountable {
/** Reloads the underlying Lucene Suggester */
public void reload(SolrCore core, SolrIndexSearcher searcher) throws IOException {
LOG.info("reload(" + name + ")");
LOG.info("SolrSuggester.reload(" + name + ")");
if (dictionary == null && storeDir != null) {
File lookupFile = getStoreFile();
if (lookupFile.exists()) {

112
solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java
View File

@ -42,11 +42,6 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
public void setUp() throws Exception {
super.setUp();
assertQ(req("qt", "standard",
"q", "*:*"),
"//*[@numFound='0']"
);
// id, cat, price, weight
assertU(adoc("id", "0", "cat", "This is a title", "price", "5", "weight", "10"));
assertU(adoc("id", "1", "cat", "This is another title", "price", "10", "weight", "10"));
@ -68,6 +63,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
super.tearDown();
assertU(delQ("*:*"));
assertU((commit()));
waitForWarming();
// rebuild suggesters with empty index
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_BUILD_ALL, "true"),
@ -81,7 +77,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
SuggesterParams.SUGGEST_DICT, "suggest_fuzzy_doc_dict",
SuggesterParams.SUGGEST_BUILD, "true",
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='exampel']/int[@name='numFound'][.='2']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='exampel']/arr[@name='suggestions']/lst[1]/str[@name='term'][.='example inputdata']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='exampel']/arr[@name='suggestions']/lst[1]/long[@name='weight'][.='45']",
@ -93,7 +89,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
SuggesterParams.SUGGEST_DICT, "suggest_fuzzy_doc_dict",
SuggesterParams.SUGGEST_BUILD, "true",
SuggesterParams.SUGGEST_Q, "Rad",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='Rad']/int[@name='numFound'][.='2']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='Rad']/arr[@name='suggestions']/lst[1]/str[@name='term'][.='Rad fox']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='Rad']/arr[@name='suggestions']/lst[1]/long[@name='weight'][.='35']",
@ -108,7 +104,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
SuggesterParams.SUGGEST_DICT, "suggest_fuzzy_doc_expr_dict",
SuggesterParams.SUGGEST_BUILD, "true",
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_expr_dict']/lst[@name='exampel']/int[@name='numFound'][.='2']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_expr_dict']/lst[@name='exampel']/arr[@name='suggestions']/lst[1]/str[@name='term'][.='example inputdata']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_expr_dict']/lst[@name='exampel']/arr[@name='suggestions']/lst[1]/long[@name='weight'][.='120']",
@ -138,7 +134,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
SuggesterParams.SUGGEST_DICT, "suggest_fuzzy_doc_expr_dict",
SuggesterParams.SUGGEST_BUILD, "true",
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='exampel']/int[@name='numFound'][.='2']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='exampel']/arr[@name='suggestions']/lst[1]/str[@name='term'][.='example inputdata']",
"//lst[@name='suggest']/lst[@name='suggest_fuzzy_doc_dict']/lst[@name='exampel']/arr[@name='suggestions']/lst[1]/long[@name='weight'][.='45']",
@ -157,7 +153,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_BUILD_ALL, "true",
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//str[@name='command'][.='buildAll']"
);
@ -172,7 +168,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_RELOAD_ALL, "true",
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//str[@name='command'][.='reloadAll']"
);
@ -189,7 +185,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
req("qt", rh,
SuggesterParams.SUGGEST_DICT, fakeSuggesterName,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
SolrException.ErrorCode.BAD_REQUEST
);
@ -197,7 +193,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
"' parameter not specified and no default suggester configured",
req("qt", rh,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
SolrException.ErrorCode.BAD_REQUEST
);
}
@ -224,7 +220,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
);
@ -232,12 +228,22 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertU(adoc("id", "10", "cat", "example data extra ", "price", "40", "weight", "35"));
assertU((commit()));
waitForWarming();
// buildOnCommit=false, this doc should not be in the suggester yet
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
);
reloadCore(random().nextBoolean());
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "3"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='3']"
);
@ -262,7 +268,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='0']"
);
@ -276,7 +282,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
);
@ -286,7 +292,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
);
@ -294,12 +300,21 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertU(adoc("id", "10", "cat", "example data extra ", "price", "40", "weight", "35"));
assertU((commit()));
waitForWarming();
// buildOnCommit=false, this doc should not be in the suggester yet
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
);
reloadCore(random().nextBoolean());
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "3"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='2']"
);
@ -313,7 +328,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "example",
SuggesterParams.SUGGEST_COUNT, "3"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='example']/int[@name='numFound'][.='3']"
);
@ -346,7 +361,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
);
@ -356,7 +371,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
);
@ -366,7 +381,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggester,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggester + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
);
}
@ -395,12 +410,14 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
// assert that buildOnStartup=false
assertEquals("false",
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='buildOnStartup']", false));
assertEquals("true",
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='buildOnCommit']", false));
// verify that this suggester is built (there was a commit in setUp)
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggesterFuzzy,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggesterFuzzy + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
);
@ -415,7 +432,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggesterFuzzy,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggesterFuzzy + "']/lst[@name='exampel']/int[@name='numFound'][.='0']"
);
@ -430,7 +447,7 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggesterFuzzy,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggesterFuzzy + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
);
@ -442,6 +459,8 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='storeDir']", false));
assertEquals("true",
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='buildOnStartup']", false));
assertEquals("false",
h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='buildOnCommit']", false));
// reload the core
reloadCore(createNewCores);
@ -449,13 +468,40 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggestStartup,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "2"),
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggestStartup + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
);
// add one more doc, this should not be seen without rebuilding manually or reloading the core (buildOnCommit=false)
assertU(adoc("id", "10", "cat", "example data extra ", "price", "40", "weight", "35"));
assertU((commit()));
waitForWarming();
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggestStartup,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggestStartup + "']/lst[@name='exampel']/int[@name='numFound'][.='2']"
);
// build the suggester manually
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggestStartup,
SuggesterParams.SUGGEST_BUILD, "true"),
"//str[@name='command'][.='build']"
);
assertQ(req("qt", rh,
SuggesterParams.SUGGEST_DICT, suggestStartup,
SuggesterParams.SUGGEST_Q, "exampel",
SuggesterParams.SUGGEST_COUNT, "5"),
"//lst[@name='suggest']/lst[@name='" + suggestStartup + "']/lst[@name='exampel']/int[@name='numFound'][.='3']"
);
}
private void reloadCore(boolean createNewCore) throws Exception {
// if (createNewCore) {
if (createNewCore) {
CoreContainer cores = h.getCoreContainer();
SolrCore core = h.getCore();
String dataDir1 = core.getDataDir();
@ -464,11 +510,11 @@ public class SuggestComponentTest extends SolrTestCaseJ4 {
SolrCore createdCore = cores.create(cd);
assertEquals(dataDir1, createdCore.getDataDir());
assertEquals(createdCore, h.getCore());
// } else {
// h.reload();
// // On regular reloading, wait until the new searcher is registered
// waitForWarming();
// }
} else {
h.reload();
// On regular reloading, wait until the new searcher is registered
waitForWarming();
}
assertQ(req("qt", "standard",
"q", "*:*"),