Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into gradle-master

This commit is contained in:
Dawid Weiss 2020-01-03 11:02:55 +01:00
parent 1a690d95ad 7d0cf0df32
commit 2bab5ea442
23 changed files with 178 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -85,7 +85,6 @@ Improvements
* SOLR-13749: New cross collection join filter (XCJF) (Dan Fox, Kevin Watters, via Gus Heck)
Other Changes
----------------------
* SOLR-10288: Remove non-minified JavaScript from the webapp. (Erik Hatcher, marcussorealheis)
@ -203,6 +202,8 @@ Bug Fixes
* SOLR-14109: Always log to stdout from server/scripts/cloud-scripts/zkcli.{bat|sh} (janhoy)
* SOLR-14122: SimUtils converts v2 to v1 request params incorrectly. (Li Cao, ab)
Other Changes
---------------------

51
solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimUtils.java
View File

@ -330,32 +330,47 @@ public class SimUtils {
private static final Map<String, String> v2v1Mapping = new HashMap<>();
static {
for (CollectionApiMapping.Meta meta : CollectionApiMapping.Meta.values()) {
if (meta.action != null) v2v1Mapping.put(meta.commandName, meta.action.toLower());
if (meta.action != null) {
String key;
if (meta.commandName != null) {
key = meta.commandName;
} else {
key = meta.action.toLower();
}
v2v1Mapping.put(key, meta.action.toLower());
} else {
log.warn("V2 action " + meta + " has no equivalent V1 action");
}
}
}
/**
* Convert a V2 {@link org.apache.solr.client.solrj.request.CollectionAdminRequest} to regular {@link org.apache.solr.common.params.SolrParams}
* @param req request
* @return payload converted to V1 params
* @return request payload and parameters converted to V1 params
*/
public static ModifiableSolrParams v2AdminRequestToV1Params(V2Request req) {
Map<String, Object> reqMap = new HashMap<>();
((V2Request)req).toMap(reqMap);
req.toMap(reqMap);
String path = (String)reqMap.get("path");
if (!path.startsWith("/c/") || path.length() < 4) {
if (!(path.startsWith("/c/") || path.startsWith("/collections/")) || path.length() < 4) {
throw new UnsupportedOperationException("Unsupported V2 request path: " + reqMap);
}
Map<String, Object> cmd = (Map<String, Object>)reqMap.get("command");
Map<String, Object> cmd;
Object cmdObj = reqMap.get("command");
if (cmdObj instanceof String) {
cmd = (Map<String, Object>)Utils.fromJSONString((String)cmdObj);
} else if (cmdObj instanceof Map) {
cmd = (Map<String, Object>)cmdObj;
} else {
throw new UnsupportedOperationException("Unsupported 'command': " + cmdObj + " (of type " + cmdObj.getClass() + ")");
}
if (cmd.size() != 1) {
throw new UnsupportedOperationException("Unsupported multi-command V2 request: " + reqMap);
}
String a = cmd.keySet().iterator().next();
ModifiableSolrParams params = new ModifiableSolrParams();
if (req.getParams() != null) {
params.add(req.getParams());
}
params.add(CollectionAdminParams.COLLECTION, path.substring(3));
params.set("path", "/admin/collections");
if (req.getParams() != null) {
params.add(req.getParams());
}
@ -363,6 +378,24 @@ public class SimUtils {
for (Map.Entry<String, Object> e : reqParams.entrySet()) {
params.add(e.getKey(), e.getValue().toString());
}
// trim the leading /
path = path.substring(1);
String[] pathEls = path.split("/");
if (pathEls.length < 2) {
throw new UnsupportedOperationException("Unsupported V2 request path: " + reqMap);
}
params.set(CollectionAdminParams.COLLECTION, pathEls[1]);
if (pathEls.length > 3) {
if (!pathEls[2].equals("shards")) {
throw new UnsupportedOperationException("Invalid V2 request path: expected 'shards' but was '" + pathEls[2] + "'");
}
if (!pathEls[3].isBlank()) {
params.set("shard", pathEls[3]);
}
}
if (pathEls.length > 4 && !pathEls[4].isBlank()) {
params.set("replica", pathEls[4]);
}
// re-map from v2 to v1 action
a = v2v1Mapping.get(a);
if (a == null) {

5
solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml
View File

@ -83,25 +83,22 @@
that match a particular query.
-->
<filterCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10" />

3
solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml
View File

@ -37,21 +37,18 @@
-->
<filterCache
enabled="${filterCache.enabled}"
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
enabled="${queryResultCache.enabled}"
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
enabled="${documentCache.enabled}"
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>

6
solr/core/src/test-files/solr/collection1/conf/solrconfig-caching.xml
View File

@ -21,19 +21,19 @@
<schemaFactory class="ClassicIndexSchemaFactory"/>
<query>
<cache name="lfuCacheDecayFalse"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="9"
timeDecay="false" />
<cache name="lfuCacheDecayTrue"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="9"
timeDecay="true" />
<cache name="lfuCacheDecayDefault"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="9" />
</query>

5
solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml
View File

@ -83,25 +83,22 @@
that match a particular query.
-->
<filterCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10" />

4
solr/core/src/test-files/solr/collection1/conf/solrconfig-deeppaging.xml
View File

@ -39,8 +39,8 @@
<!-- deep paging better play nice with caching -->
<query>
<!-- no wautowarming, it screws up our ability to sanity check cache stats in tests -->
<filterCache class="solr.CaffeineCache" size="50" initialSize="50" autowarmCount="0"/>
<queryResultCache class="solr.CaffeineCache" size="50" initialSize="50" autowarmCount="0"/>
<filterCache size="50" initialSize="50" autowarmCount="0"/>
<queryResultCache size="50" initialSize="50" autowarmCount="0"/>
<queryResultWindowSize>50</queryResultWindowSize>
<queryResultMaxDocsCached>500</queryResultMaxDocsCached>
<!-- randomized so we excersize cursors using various paths in SolrIndexSearcher -->

3
solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml
View File

@ -41,19 +41,16 @@
<query>
<filterCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<queryResultCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<documentCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>

3
solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml
View File

@ -56,19 +56,16 @@
that match a particular query.
-->
<filterCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="256"/>
<queryResultCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="1024"/>
<documentCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>

3
solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml
View File

@ -37,21 +37,18 @@
<query>
<filterCache
enabled="${filterCache.enabled:false}"
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
enabled="${queryResultCache.enabled:false}"
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
enabled="${documentCache.enabled:false}"
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>

5
solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml
View File

@ -95,25 +95,22 @@
that match a particular query.
-->
<filterCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10" />

5
solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml
View File

@ -82,25 +82,22 @@
that match a particular query.
-->
<filterCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10" />

3
solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml
View File

@ -35,8 +35,7 @@
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
<documentCache class="solr.CaffeineCache"
size="512"
<documentCache size="512"
initialSize="512"
autowarmCount="0"/>
</query>

2
solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml
View File

@ -39,4 +39,4 @@
autowarmCount="100%"
regenerator="solr.NoOpRegenerator"/>
</query>
</config>
</config>

8
solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
View File

@ -140,23 +140,23 @@
<query>
<filterCache class="solr.CaffeineCache"
<filterCache
size="512"
initialSize="512"
autowarmCount="0" />
<queryResultCache class="solr.CaffeineCache"
<queryResultCache
size="512"
initialSize="512"
autowarmCount="0" />
<documentCache class="solr.CaffeineCache"
<documentCache
size="512"
initialSize="512"
autowarmCount="0" />
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10"

6
solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
View File

@ -95,25 +95,22 @@
that match a particular query.
-->
<filterCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
class="solr.search.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10" />
@ -125,7 +122,6 @@
<!--
<cache name="myUserCache"
class="solr.search.CaffeineCache"
size="4096"
initialSize="1024"
autowarmCount="1024"

3
solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml
View File

@ -36,19 +36,16 @@
<query>
<filterCache
class="solr.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<queryResultCache
class="solr.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<documentCache
class="solr.CaffeineCache"
size="512"
initialSize="512"
autowarmCount="0"/>

14
solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
View File

@ -404,8 +404,7 @@
to occupy. Note that when this option is specified, the size
and initialSize parameters are ignored.
-->
<filterCache class="solr.CaffeineCache"
size="512"
<filterCache size="512"
initialSize="512"
autowarmCount="0"/>
@ -417,8 +416,7 @@
maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
to occupy
-->
<queryResultCache class="solr.CaffeineCache"
size="512"
<queryResultCache size="512"
initialSize="512"
autowarmCount="0"/>
@ -428,14 +426,13 @@
document). Since Lucene internal document ids are transient,
this cache will not be autowarmed.
-->
<documentCache class="solr.CaffeineCache"
size="512"
<documentCache size="512"
initialSize="512"
autowarmCount="0"/>
<!-- custom cache currently used by block join -->
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10"
@ -448,8 +445,7 @@
even if not configured here.
-->
<!--
<fieldValueCache class="solr.CaffeineCache"
size="512"
<fieldValueCache size="512"
autowarmCount="128"
showItems="32" />
-->

8
solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
View File

@ -56,19 +56,19 @@
</updateHandler>
<query>
<filterCache class="solr.CaffeineCache"
<filterCache
size="0"
initialSize="0"
autowarmCount="0"/>
<queryResultCache class="solr.CaffeineCache"
<queryResultCache
size="0"
initialSize="0"
autowarmCount="0"/>
<documentCache class="solr.CaffeineCache"
<documentCache
size="0"
initialSize="0"
autowarmCount="0"/>
<fieldValueCache class="solr.CaffeineCache"
<fieldValueCache
size="0"
autowarmCount="0"
showItems="0" />

6
solr/core/src/test-files/solr/configsets/xcjf/conf/solrconfig.xml
View File

@ -26,17 +26,17 @@
<requestHandler name="/select" class="solr.SearchHandler" />
<query>
<filterCache class="solr.CaffeineCache"
<filterCache
size="512"
initialSize="512"
autowarmCount="0" />
<queryResultCache class="solr.CaffeineCache"
<queryResultCache
size="512"
initialSize="512"
autowarmCount="0" />
<documentCache class="solr.CaffeineCache"
<documentCache
size="512"
initialSize="512"
autowarmCount="0" />

95
solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimUtils.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud.autoscaling.sim;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.common.params.SolrParams;
import org.junit.Test;
/**
*
*/
public class TestSimUtils extends SolrTestCaseJ4 {
@Test
public void testV2toV1() throws Exception {
// valid patterns
V2Request req = new V2Request.Builder("/c/myCollection")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}")
.withParams(params("foo", "bar"))
.build();
SolrParams params = SimUtils.v2AdminRequestToV1Params(req);
assertEquals("/admin/collections", params.get("path"));
assertEquals("myCollection", params.get("collection"));
assertEquals("bar", params.get("foo"));
assertEquals("addreplica", params.get("action"));
assertEquals("shard2", params.get("shard"));
assertEquals("node1:1234", params.get("node"));
assertEquals("TLOG", params.get("type"));
req = new V2Request.Builder("/c/myCollection/shards/shard1")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}")
.build();
params = SimUtils.v2AdminRequestToV1Params(req);
assertEquals("/admin/collections", params.get("path"));
assertEquals("myCollection", params.get("collection"));
// XXX should path parameters override the payload, or the other way around?
assertEquals("shard1", params.get("shard"));
req = new V2Request.Builder("/c/myCollection/shards/shard1/core_node5")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{'deletereplica':{}}")
.build();
params = SimUtils.v2AdminRequestToV1Params(req);
assertEquals("/admin/collections", params.get("path"));
assertEquals("myCollection", params.get("collection"));
// XXX should path parameters override the payload, or the other way around?
assertEquals("shard1", params.get("shard"));
assertEquals("core_node5", params.get("replica"));
// invalid patterns
req = new V2Request.Builder("/invalid/myCollection")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}")
.withParams(params("foo", "bar"))
.build();
try {
params = SimUtils.v2AdminRequestToV1Params(req);
} catch (UnsupportedOperationException e) {
// expected
assertTrue(e.toString(), e.toString().contains("request path"));
}
req = new V2Request.Builder("/collections/myCollection/foobar/xyz")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}")
.withParams(params("foo", "bar"))
.build();
try {
params = SimUtils.v2AdminRequestToV1Params(req);
} catch (UnsupportedOperationException e) {
// expected
assertTrue(e.toString(), e.toString().contains("expected 'shards'"));
}
}
}

14
solr/server/solr/configsets/_default/conf/solrconfig.xml
View File

@ -404,8 +404,7 @@
to occupy. Note that when this option is specified, the size
and initialSize parameters are ignored.
-->
<filterCache class="solr.CaffeineCache"
size="512"
<filterCache size="512"
initialSize="512"
autowarmCount="0"/>
@ -417,8 +416,7 @@
maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
to occupy
-->
<queryResultCache class="solr.CaffeineCache"
size="512"
<queryResultCache size="512"
initialSize="512"
autowarmCount="0"/>
@ -428,14 +426,13 @@
document). Since Lucene internal document ids are transient,
this cache will not be autowarmed.
-->
<documentCache class="solr.CaffeineCache"
size="512"
<documentCache size="512"
initialSize="512"
autowarmCount="0"/>
<!-- custom cache currently used by block join -->
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10"
@ -448,8 +445,7 @@
even if not configured here.
-->
<!--
<fieldValueCache class="solr.CaffeineCache"
size="512"
<fieldValueCache size="512"
autowarmCount="128"
showItems="32" />
-->

16
solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
View File

@ -428,8 +428,7 @@
to occupy. Note that when this option is specified, the size
and initialSize parameters are ignored.
-->
<filterCache class="solr.CaffeineCache"
size="512"
<filterCache size="512"
initialSize="512"
autowarmCount="0"/>
@ -441,8 +440,7 @@
maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
to occupy
-->
<queryResultCache class="solr.CaffeineCache"
size="512"
<queryResultCache size="512"
initialSize="512"
autowarmCount="0"/>
@ -452,14 +450,13 @@
document). Since Lucene internal document ids are transient,
this cache will not be autowarmed.
-->
<documentCache class="solr.CaffeineCache"
size="512"
<documentCache size="512"
initialSize="512"
autowarmCount="0"/>
<!-- custom cache currently used by block join -->
<cache name="perSegFilter"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="10"
initialSize="0"
autowarmCount="10"
@ -472,8 +469,7 @@
even if not configured here.
-->
<!--
<fieldValueCache class="solr.CaffeineCache"
size="512"
<fieldValueCache size="512"
autowarmCount="128"
showItems="32" />
-->
@ -489,7 +485,7 @@
https://lucene.apache.org/solr/guide/learning-to-rank.html
-->
<cache enable="${solr.ltr.enabled:false}" name="QUERY_DOC_FV"
class="solr.search.CaffeineCache"
class="solr.CaffeineCache"
size="4096"
initialSize="2048"
autowarmCount="4096"