diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ccbbea1a9dd..d47c57e66ec 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -162,6 +162,8 @@ Bug Fixes * SOLR-9179: NPE in IndexSchema using IBM JDK (noble, Colvin Cowie) +* SOLR-9397: Config API does not support adding caches (noble) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index eb3aa5fc7f1..653c612fe65 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -28,7 +28,17 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -49,6 +59,7 @@ import org.apache.solr.schema.IndexSchemaFactory; import org.apache.solr.search.CacheConfig; import org.apache.solr.search.FastLRUCache; import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.SolrCache; import org.apache.solr.search.ValueSourceParser; import org.apache.solr.search.stats.StatsCache; import org.apache.solr.servlet.SolrRequestParsers; @@ -91,7 +102,7 @@ public class SolrConfig extends Config implements MapSerializable { public static final String DEFAULT_CONF_FILE = "solrconfig.xml"; private RequestParams requestParams; - public static enum PluginOpts { + public enum PluginOpts { MULTI_OK, REQUIRE_NAME, REQUIRE_NAME_IN_OVERLAY, @@ -254,7 +265,6 @@ public class SolrConfig extends Config implements MapSerializable { dataDir = get("dataDir", null); if (dataDir != null && dataDir.length() == 0) dataDir = null; - userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache"); org.apache.solr.search.SolrIndexSearcher.initRegenerators(this); @@ -276,6 +286,16 @@ public class SolrConfig extends Config implements MapSerializable { maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE); slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1); for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin); + + Map userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache"); + List caches = getPluginInfos(SolrCache.class.getName()); + if (!caches.isEmpty()) { + for (PluginInfo c : caches) { + userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null)); + } + } + this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs); + updateHandlerInfo = loadUpdatehandlerInfo(); multipartUploadLimitKB = getInt( @@ -317,6 +337,7 @@ public class SolrConfig extends Config implements MapSerializable { .add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) .add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) .add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) + .add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) // TODO: WTF is up with queryConverter??? // it apparently *only* works as a singleton? - SOLR-4304 // and even then -- only if there is a single SpellCheckComponent @@ -457,7 +478,7 @@ public class SolrConfig extends Config implements MapSerializable { public final CacheConfig queryResultCacheConfig; public final CacheConfig documentCacheConfig; public final CacheConfig fieldValueCacheConfig; - public final CacheConfig[] userCacheConfigs; + public final Map userCacheConfigs; // SolrIndexSearcher - more... public final boolean useFilterForSortedQuery; public final int queryResultWindowSize; diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java index 40e54dccb4e..ee333f8787b 100644 --- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java +++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java @@ -17,10 +17,10 @@ package org.apache.solr.search; import javax.xml.xpath.XPathConstants; - import java.lang.invoke.MethodHandles; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -75,14 +75,15 @@ public class CacheConfig implements MapSerializable{ this.regenerator = regenerator; } - public static CacheConfig[] getMultipleConfigs(SolrConfig solrConfig, String configPath) { - NodeList nodes = (NodeList)solrConfig.evaluate(configPath, XPathConstants.NODESET); - if (nodes==null || nodes.getLength()==0) return null; - CacheConfig[] configs = new CacheConfig[nodes.getLength()]; - for (int i=0; i getMultipleConfigs(SolrConfig solrConfig, String configPath) { + NodeList nodes = (NodeList) solrConfig.evaluate(configPath, XPathConstants.NODESET); + if (nodes == null || nodes.getLength() == 0) return new LinkedHashMap<>(); + Map result = new HashMap<>(nodes.getLength()); + for (int i = 0; i < nodes.getLength(); i++) { + CacheConfig config = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()), configPath); + result.put(config.args.get(NAME), config); } - return configs; + return result; } @@ -101,9 +102,14 @@ public class CacheConfig implements MapSerializable{ public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map attrs, String xpath) { CacheConfig config = new CacheConfig(); config.nodeName = nodeName; + Map attrsCopy = new LinkedHashMap<>(attrs.size()); + for (Map.Entry e : attrs.entrySet()) { + attrsCopy.put(e.getKey(), String.valueOf(e.getValue())); + } + attrs = attrsCopy; config.args = attrs; - Map map = solrConfig.getOverlay().getEditableSubProperties(xpath); + Map map = xpath == null ? null : solrConfig.getOverlay().getEditableSubProperties(xpath); if(map != null){ HashMap mapCopy = new HashMap<>(config.args); for (Map.Entry e : map.entrySet()) { diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index cc719f0be37..0f480c67579 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -36,62 +36,16 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.collect.Iterables; import org.apache.lucene.document.Document; import org.apache.lucene.document.DocumentStoredFieldVisitor; import org.apache.lucene.document.LazyDocument; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.ExitableDirectoryReader; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiPostingsEnum; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.*; import org.apache.lucene.index.StoredFieldVisitor.Status; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CollectionStatistics; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.EarlyTerminatingSortingCollector; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MultiCollector; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SimpleCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermStatistics; -import org.apache.lucene.search.TimeLimitingCollector; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopDocsCollector; -import org.apache.lucene.search.TopFieldCollector; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TopScoreDocCollector; -import org.apache.lucene.search.TotalHitCountCollector; -import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -128,10 +82,6 @@ import org.apache.solr.update.SolrIndexConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.base.Objects; -import com.google.common.collect.Iterables; - /** * SolrIndexSearcher adds schema awareness and caching functionality over {@link IndexSearcher}. * @@ -337,13 +287,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance(); if (documentCache != null) clist.add(documentCache); - if (solrConfig.userCacheConfigs == null) { + if (solrConfig.userCacheConfigs.isEmpty()) { cacheMap = NO_GENERIC_CACHES; } else { - cacheMap = new HashMap<>(solrConfig.userCacheConfigs.length); - for (CacheConfig userCacheConfig : solrConfig.userCacheConfigs) { - SolrCache cache = null; - if (userCacheConfig != null) cache = userCacheConfig.newInstance(); + cacheMap = new HashMap<>(solrConfig.userCacheConfigs.size()); + for (Map.Entry e : solrConfig.userCacheConfigs.entrySet()) { + SolrCache cache = e.getValue().newInstance(); if (cache != null) { cacheMap.put(cache.name(), cache); clist.add(cache); diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java index 250ccf091fe..9b71ff819d2 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java @@ -22,6 +22,7 @@ import java.io.StringReader; import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -35,8 +36,12 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.DumpRequestHandler; import org.apache.solr.handler.TestBlobHandler; import org.apache.solr.handler.TestSolrConfigHandlerConcurrent; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrCache; import org.apache.solr.util.RestTestBase; import org.apache.solr.util.RestTestHarness; import org.eclipse.jetty.servlet.ServletHolder; @@ -449,7 +454,56 @@ public class TestSolrConfigHandler extends RestTestBase { assertEquals(2, initArgs.size()); assertTrue(((Map)initArgs.get(0)).containsKey("suggester")); assertTrue(((Map)initArgs.get(1)).containsKey("suggester")); - System.out.println(map); + + payload = "{\n" + + "'add-requesthandler' : { 'name' : '/dump101', 'class': " + + "'" + CacheTest.class.getName() + "' " + + ", 'startup' : 'lazy'}\n" + + "}"; + runConfigCommand(writeHarness, "/config?wt=json", payload); + + testForResponseElement(writeHarness, + testServerBaseUrl, + "/config/overlay?wt=json", + cloudSolrClient, + Arrays.asList("overlay", "requestHandler", "/dump101", "startup"), + "lazy", + 10); + + payload = "{\n" + + "'add-cache' : {name:'lfuCacheDecayFalse', class:'solr.search.LFUCache', size:10 ,initialSize:9 , timeDecay:false }," + + "'add-cache' : {name: 'perSegFilter', class: 'solr.search.LRUCache', size:10, initialSize:0 , autowarmCount:10}}"; + runConfigCommand(writeHarness, "/config?wt=json", payload); + + map = testForResponseElement(writeHarness, + testServerBaseUrl, + "/config/overlay?wt=json", + cloudSolrClient, + Arrays.asList("overlay", "cache", "lfuCacheDecayFalse", "class"), + "solr.search.LFUCache", + 10); + assertEquals("solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of("overlay", "cache", "perSegFilter", "class"))); + + map = getRespMap("/dump101?cacheNames=lfuCacheDecayFalse&cacheNames=perSegFilter&wt=json", writeHarness); + assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "perSegFilter"))); + assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LFUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "lfuCacheDecayFalse"))); + + } + + public static class CacheTest extends DumpRequestHandler { + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + super.handleRequestBody(req, rsp); + String[] caches = req.getParams().getParams("cacheNames"); + if(caches != null && caches.length>0){ + HashMap m = new HashMap(); + rsp.add("caches", m); + for (String c : caches) { + SolrCache cache = req.getSearcher().getCache(c); + if(cache != null) m.put(c, cache.getClass().getName()); + } + } + } } public static Map testForResponseElement(RestTestHarness harness,