Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9397: Config API does not support adding caches

This commit is contained in:
Noble Paul 2016-08-10 20:23:22 +05:30
parent dd03d39dd6
commit 64c99293d7
5 changed files with 106 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -196,6 +196,8 @@ Bug Fixes
* SOLR-9179: NPE in IndexSchema using IBM JDK (noble, Colvin Cowie)
* SOLR-9397: Config API does not support adding caches (noble)
Optimizations
----------------------

29
solr/core/src/java/org/apache/solr/core/SolrConfig.java
View File

@ -28,7 +28,17 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -49,6 +59,7 @@ import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.search.CacheConfig;
import org.apache.solr.search.FastLRUCache;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.SolrCache;
import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.search.stats.StatsCache;
import org.apache.solr.servlet.SolrRequestParsers;
@ -91,7 +102,7 @@ public class SolrConfig extends Config implements MapSerializable {
public static final String DEFAULT_CONF_FILE = "solrconfig.xml";
private RequestParams requestParams;
public static enum PluginOpts {
public enum PluginOpts {
MULTI_OK,
REQUIRE_NAME,
REQUIRE_NAME_IN_OVERLAY,
@ -254,7 +265,6 @@ public class SolrConfig extends Config implements MapSerializable {
dataDir = get("dataDir", null);
if (dataDir != null && dataDir.length() == 0) dataDir = null;
userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");
org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);
@ -276,6 +286,16 @@ public class SolrConfig extends Config implements MapSerializable {
maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);
slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
Map<String, CacheConfig> userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");
List<PluginInfo> caches = getPluginInfos(SolrCache.class.getName());
if (!caches.isEmpty()) {
for (PluginInfo c : caches) {
userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null));
}
}
this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs);
updateHandlerInfo = loadUpdatehandlerInfo();
multipartUploadLimitKB = getInt(
@ -317,6 +337,7 @@ public class SolrConfig extends Config implements MapSerializable {
.add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
// TODO: WTF is up with queryConverter???
// it apparently *only* works as a singleton? - SOLR-4304
// and even then -- only if there is a single SpellCheckComponent
@ -457,7 +478,7 @@ public class SolrConfig extends Config implements MapSerializable {
public final CacheConfig queryResultCacheConfig;
public final CacheConfig documentCacheConfig;
public final CacheConfig fieldValueCacheConfig;
public final CacheConfig[] userCacheConfigs;
public final Map<String, CacheConfig> userCacheConfigs;
// SolrIndexSearcher - more...
public final boolean useFilterForSortedQuery;
public final int queryResultWindowSize;

20
solr/core/src/java/org/apache/solr/search/CacheConfig.java
View File

@ -17,10 +17,10 @@
package org.apache.solr.search;
import javax.xml.xpath.XPathConstants;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -75,14 +75,15 @@ public class CacheConfig implements MapSerializable{
this.regenerator = regenerator;
}
public static CacheConfig[] getMultipleConfigs(SolrConfig solrConfig, String configPath) {
public static Map<String, CacheConfig> getMultipleConfigs(SolrConfig solrConfig, String configPath) {
NodeList nodes = (NodeList) solrConfig.evaluate(configPath, XPathConstants.NODESET);
if (nodes==null || nodes.getLength()==0) return null;
CacheConfig[] configs = new CacheConfig[nodes.getLength()];
if (nodes == null || nodes.getLength() == 0) return new LinkedHashMap<>();
Map<String, CacheConfig> result = new HashMap<>(nodes.getLength());
for (int i = 0; i < nodes.getLength(); i++) {
configs[i] = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()), configPath);
CacheConfig config = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()), configPath);
result.put(config.args.get(NAME), config);
}
return configs;
return result;
}
@ -101,9 +102,14 @@ public class CacheConfig implements MapSerializable{
public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map<String,String> attrs, String xpath) {
CacheConfig config = new CacheConfig();
config.nodeName = nodeName;
Map attrsCopy = new LinkedHashMap<>(attrs.size());
for (Map.Entry<String, String> e : attrs.entrySet()) {
attrsCopy.put(e.getKey(), String.valueOf(e.getValue()));
}
attrs = attrsCopy;
config.args = attrs;
Map<String, String> map = solrConfig.getOverlay().getEditableSubProperties(xpath);
Map<String, String> map = xpath == null ? null : solrConfig.getOverlay().getEditableSubProperties(xpath);
if(map != null){
HashMap<String, String> mapCopy = new HashMap<>(config.args);
for (Map.Entry<String, String> e : map.entrySet()) {

69
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -36,62 +36,16 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.collect.Iterables;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.document.LazyDocument;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiPostingsEnum;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.*;
import org.apache.lucene.index.StoredFieldVisitor.Status;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.EarlyTerminatingSortingCollector;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MultiCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TimeLimitingCollector;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@ -128,10 +82,6 @@ import org.apache.solr.update.SolrIndexConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.collect.Iterables;
/**
* SolrIndexSearcher adds schema awareness and caching functionality over {@link IndexSearcher}.
*
@ -337,13 +287,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
if (documentCache != null) clist.add(documentCache);
if (solrConfig.userCacheConfigs == null) {
if (solrConfig.userCacheConfigs.isEmpty()) {
cacheMap = NO_GENERIC_CACHES;
} else {
cacheMap = new HashMap<>(solrConfig.userCacheConfigs.length);
for (CacheConfig userCacheConfig : solrConfig.userCacheConfigs) {
SolrCache cache = null;
if (userCacheConfig != null) cache = userCacheConfig.newInstance();
cacheMap = new HashMap<>(solrConfig.userCacheConfigs.size());
for (Map.Entry<String,CacheConfig> e : solrConfig.userCacheConfigs.entrySet()) {
SolrCache cache = e.getValue().newInstance();
if (cache != null) {
cacheMap.put(cache.name(), cache);
clist.add(cache);

56
solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
View File

@ -22,6 +22,7 @@ import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -35,8 +36,12 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.handler.DumpRequestHandler;
import org.apache.solr.handler.TestBlobHandler;
import org.apache.solr.handler.TestSolrConfigHandlerConcurrent;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrCache;
import org.apache.solr.util.RestTestBase;
import org.apache.solr.util.RestTestHarness;
import org.eclipse.jetty.servlet.ServletHolder;
@ -449,7 +454,56 @@ public class TestSolrConfigHandler extends RestTestBase {
assertEquals(2, initArgs.size());
assertTrue(((Map)initArgs.get(0)).containsKey("suggester"));
assertTrue(((Map)initArgs.get(1)).containsKey("suggester"));
System.out.println(map);
payload = "{\n" +
"'add-requesthandler' : { 'name' : '/dump101', 'class': " +
"'" + CacheTest.class.getName() + "' " +
", 'startup' : 'lazy'}\n" +
"}";
runConfigCommand(writeHarness, "/config?wt=json", payload);
testForResponseElement(writeHarness,
testServerBaseUrl,
"/config/overlay?wt=json",
cloudSolrClient,
Arrays.asList("overlay", "requestHandler", "/dump101", "startup"),
"lazy",
10);
payload = "{\n" +
"'add-cache' : {name:'lfuCacheDecayFalse', class:'solr.search.LFUCache', size:10 ,initialSize:9 , timeDecay:false }," +
"'add-cache' : {name: 'perSegFilter', class: 'solr.search.LRUCache', size:10, initialSize:0 , autowarmCount:10}}";
runConfigCommand(writeHarness, "/config?wt=json", payload);
map = testForResponseElement(writeHarness,
testServerBaseUrl,
"/config/overlay?wt=json",
cloudSolrClient,
Arrays.asList("overlay", "cache", "lfuCacheDecayFalse", "class"),
"solr.search.LFUCache",
10);
assertEquals("solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of("overlay", "cache", "perSegFilter", "class")));
map = getRespMap("/dump101?cacheNames=lfuCacheDecayFalse&cacheNames=perSegFilter&wt=json", writeHarness);
assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "perSegFilter")));
assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LFUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "lfuCacheDecayFalse")));
}
public static class CacheTest extends DumpRequestHandler {
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
super.handleRequestBody(req, rsp);
String[] caches = req.getParams().getParams("cacheNames");
if(caches != null && caches.length>0){
HashMap m = new HashMap();
rsp.add("caches", m);
for (String c : caches) {
SolrCache cache = req.getSearcher().getCache(c);
if(cache != null) m.put(c, cache.getClass().getName());
}
}
}
}
public static Map testForResponseElement(RestTestHarness harness,