From 84d8385fb9fa3f794d5e473219ca6b5b2abee5f2 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Thu, 10 Aug 2017 07:09:48 -0400 Subject: [PATCH 01/11] LUCENE-7918: Fix incorrect equals method. --- .../apache/lucene/spatial3d/geom/GeoBaseCompositeShape.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCompositeShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCompositeShape.java index 16a507c14a4..5fe8af154ee 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCompositeShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCompositeShape.java @@ -108,7 +108,7 @@ public abstract class GeoBaseCompositeShape implements GeoSh @Override public int hashCode() { - return super.hashCode() * 31 + shapes.hashCode();//TODO cache + return shapes.hashCode(); } @Override @@ -116,6 +116,6 @@ public abstract class GeoBaseCompositeShape implements GeoSh if (!(o instanceof GeoBaseCompositeShape)) return false; GeoBaseCompositeShape other = (GeoBaseCompositeShape) o; - return super.equals(o) && shapes.equals(other.shapes); + return shapes.equals(other.shapes); } } From 9b0fccfbda7387f0cb21967d6215f1dbc0f78b65 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 8 Aug 2017 09:19:24 +0100 Subject: [PATCH 02/11] Remove (package) javadocs in solrj UuidEvaluator. --- .../org/apache/solr/client/solrj/io/eval/UuidEvaluator.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UuidEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UuidEvaluator.java index 88acee45f37..0f22ff26502 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UuidEvaluator.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UuidEvaluator.java @@ -14,9 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * - */ package org.apache.solr.client.solrj.io.eval; import java.io.IOException; From 5bdd017e73e203539b7b5f94fa3c109dc5bbaea5 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 8 Aug 2017 09:20:12 +0100 Subject: [PATCH 03/11] Fix trivial warning in LowerCaseTokenizerFactory. --- .../apache/lucene/analysis/core/LowerCaseTokenizerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java index a3e06c7a608..44e27429b63 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java @@ -68,7 +68,7 @@ public class LowerCaseTokenizerFactory extends TokenizerFactory implements Multi @Override public AbstractAnalysisFactory getMultiTermComponent() { - Map map = new HashMap<>(getOriginalArgs()); + Map map = new HashMap<>(getOriginalArgs()); map.remove("maxTokenLen"); //removing "maxTokenLen" argument for LowerCaseFilterFactory init return new LowerCaseFilterFactory(map); } From ff25225fc37d4497a373820787ea6a1b2890c878 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 8 Aug 2017 09:21:05 +0100 Subject: [PATCH 04/11] Fix warnings in BlendedInfixSuggesterTest. --- .../suggest/analyzing/BlendedInfixSuggesterTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java index fe14e23815d..ace44678957 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java @@ -27,6 +27,7 @@ import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.suggest.Input; import org.apache.lucene.search.suggest.InputArrayIterator; @@ -302,17 +303,17 @@ public class BlendedInfixSuggesterTest extends LuceneTestCase { assertEquals(2, responses.size()); - responses = suggester.lookup(term, (Map) null, 1, false, false); + responses = suggester.lookup(term, (Map) null, 1, false, false); assertEquals(1, responses.size()); - responses = suggester.lookup(term, (Map) null, 2, false, false); + responses = suggester.lookup(term, (Map) null, 2, false, false); assertEquals(2, responses.size()); - responses = suggester.lookup(term, (Set) null, 1, false, false); + responses = suggester.lookup(term, (Set) null, 1, false, false); assertEquals(1, responses.size()); - responses = suggester.lookup(term, (Set) null, 2, false, false); + responses = suggester.lookup(term, (Set) null, 2, false, false); assertEquals(2, responses.size()); From 3f3a71ad65f0c4de111aaf77ddd9c48ba0f1ea96 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 10 Aug 2017 19:43:52 +0100 Subject: [PATCH 05/11] SOLR-11223: do asserts on all 8 results, test that feature-vector retrieval does not alter score --- .../apache/solr/ltr/TestLTROnSolrCloud.java | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java index a26fdbac25a..8c3950a36b7 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java @@ -91,6 +91,19 @@ public class TestLTROnSolrCloud extends TestRerankBase { assertEquals("2", queryResponse.getResults().get(1).get("id").toString()); assertEquals("3", queryResponse.getResults().get(2).get("id").toString()); assertEquals("4", queryResponse.getResults().get(3).get("id").toString()); + assertEquals("5", queryResponse.getResults().get(4).get("id").toString()); + assertEquals("6", queryResponse.getResults().get(5).get("id").toString()); + assertEquals("7", queryResponse.getResults().get(6).get("id").toString()); + assertEquals("8", queryResponse.getResults().get(7).get("id").toString()); + + final Float original_result0_score = (Float)queryResponse.getResults().get(0).get("score"); + final Float original_result1_score = (Float)queryResponse.getResults().get(1).get("score"); + final Float original_result2_score = (Float)queryResponse.getResults().get(2).get("score"); + final Float original_result3_score = (Float)queryResponse.getResults().get(3).get("score"); + final Float original_result4_score = (Float)queryResponse.getResults().get(4).get("score"); + final Float original_result5_score = (Float)queryResponse.getResults().get(5).get("score"); + final Float original_result6_score = (Float)queryResponse.getResults().get(6).get("score"); + final Float original_result7_score = (Float)queryResponse.getResults().get(7).get("score"); final String result0_features= FeatureLoggerTestUtils.toFeatureVector( "powpularityS","64.0", "c3","2.0"); @@ -100,9 +113,40 @@ public class TestLTROnSolrCloud extends TestRerankBase { "powpularityS","36.0", "c3","2.0"); final String result3_features= FeatureLoggerTestUtils.toFeatureVector( "powpularityS","25.0", "c3","2.0"); + final String result4_features= FeatureLoggerTestUtils.toFeatureVector( + "powpularityS","16.0", "c3","2.0"); + final String result5_features= FeatureLoggerTestUtils.toFeatureVector( + "powpularityS", "9.0", "c3","2.0"); + final String result6_features= FeatureLoggerTestUtils.toFeatureVector( + "powpularityS", "4.0", "c3","2.0"); + final String result7_features= FeatureLoggerTestUtils.toFeatureVector( + "powpularityS", "1.0", "c3","2.0"); - // Test re-rank and feature vectors returned + + // Test feature vectors returned (without re-ranking) query.setFields("*,score,features:[fv]"); + queryResponse = + solrCluster.getSolrClient().query(COLLECTION,query); + assertEquals(8, queryResponse.getResults().getNumFound()); + assertEquals("1", queryResponse.getResults().get(0).get("id").toString()); + assertEquals("2", queryResponse.getResults().get(1).get("id").toString()); + assertEquals("3", queryResponse.getResults().get(2).get("id").toString()); + assertEquals("4", queryResponse.getResults().get(3).get("id").toString()); + assertEquals("5", queryResponse.getResults().get(4).get("id").toString()); + assertEquals("6", queryResponse.getResults().get(5).get("id").toString()); + assertEquals("7", queryResponse.getResults().get(6).get("id").toString()); + assertEquals("8", queryResponse.getResults().get(7).get("id").toString()); + + assertEquals(original_result0_score, queryResponse.getResults().get(0).get("score")); + assertEquals(original_result1_score, queryResponse.getResults().get(1).get("score")); + assertEquals(original_result2_score, queryResponse.getResults().get(2).get("score")); + assertEquals(original_result3_score, queryResponse.getResults().get(3).get("score")); + assertEquals(original_result4_score, queryResponse.getResults().get(4).get("score")); + assertEquals(original_result5_score, queryResponse.getResults().get(5).get("score")); + assertEquals(original_result6_score, queryResponse.getResults().get(6).get("score")); + assertEquals(original_result7_score, queryResponse.getResults().get(7).get("score")); + + // Test feature vectors returned (with re-ranking) query.add("rq", "{!ltr model=powpularityS-model reRankDocs=8}"); queryResponse = solrCluster.getSolrClient().query(COLLECTION,query); @@ -119,6 +163,18 @@ public class TestLTROnSolrCloud extends TestRerankBase { assertEquals("5", queryResponse.getResults().get(3).get("id").toString()); assertEquals(result3_features, queryResponse.getResults().get(3).get("features").toString()); + assertEquals("4", queryResponse.getResults().get(4).get("id").toString()); + assertEquals(result4_features, + queryResponse.getResults().get(4).get("features").toString()); + assertEquals("3", queryResponse.getResults().get(5).get("id").toString()); + assertEquals(result5_features, + queryResponse.getResults().get(5).get("features").toString()); + assertEquals("2", queryResponse.getResults().get(6).get("id").toString()); + assertEquals(result6_features, + queryResponse.getResults().get(6).get("features").toString()); + assertEquals("1", queryResponse.getResults().get(7).get("id").toString()); + assertEquals(result7_features, + queryResponse.getResults().get(7).get("features").toString()); } private void setupSolrCluster(int numShards, int numReplicas, int numServers, int maxShardsPerNode) throws Exception { From b3127ec582a9acebc78d6c45f44516cb585b9540 Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Thu, 10 Aug 2017 21:55:02 -0400 Subject: [PATCH 06/11] SOLR-11225: Add cumulativeProbability Stream Evaluator --- .../apache/solr/handler/StreamHandler.java | 1 + .../eval/CumulativeProbabilityEvaluator.java | 67 +++++++++++++++++++ .../solrj/io/stream/StreamExpressionTest.java | 17 +++++ 3 files changed, 85 insertions(+) create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java index 687eb41b6aa..399c92bb7a0 100644 --- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java @@ -224,6 +224,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware, .withFunctionName("kolmogorovSmirnov", KolmogorovSmirnovEvaluator.class) .withFunctionName("ks", KolmogorovSmirnovEvaluator.class) .withFunctionName("asc", AscEvaluator.class) + .withFunctionName("cumulativeProbability", CumulativeProbabilityEvaluator.class) // Boolean Stream Evaluators .withFunctionName("and", AndEvaluator.class) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java new file mode 100644 index 00000000000..8d61294dff3 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.io.eval; + +import java.io.IOException; +import java.util.Locale; + +import org.apache.commons.math3.distribution.RealDistribution; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.expr.Explanation; +import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType; +import org.apache.solr.client.solrj.io.stream.expr.Expressible; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter; +import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; + +public class CumulativeProbabilityEvaluator extends ComplexEvaluator implements Expressible { + + private static final long serialVersionUID = 1; + + public CumulativeProbabilityEvaluator(StreamExpression expression, StreamFactory factory) throws IOException { + super(expression, factory); + + if(2 != subEvaluators.size()){ + throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting two values (regression result and a number) but found %d",expression,subEvaluators.size())); + } + } + + public Number evaluate(Tuple tuple) throws IOException { + StreamEvaluator a = subEvaluators.get(0); + StreamEvaluator b = subEvaluators.get(1); + Number number1 = (Number)b.evaluate(tuple); + RealDistribution rd= (RealDistribution)a.evaluate(tuple); + + return rd.cumulativeProbability(number1.doubleValue()); + } + + @Override + public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException { + StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass())); + return expression; + } + + @Override + public Explanation toExplanation(StreamFactory factory) throws IOException { + return new Explanation(nodeId.toString()) + .withExpressionType(ExpressionType.EVALUATOR) + .withFunctionName(factory.getFunctionName(getClass())) + .withImplementingClass(getClass().getName()) + .withExpression(toExpression(factory).toString()); + } +} \ No newline at end of file diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java index 93e52871d6a..14632b45f6f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java @@ -5287,6 +5287,23 @@ public class StreamExpressionTest extends SolrCloudTestCase { } + @Test + public void testCumulativeProbability() throws Exception { + String expr = "cumulativeProbability(normalDistribution(500, 40), 500)"; + ModifiableSolrParams paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", expr); + paramsLoc.set("qt", "/stream"); + + String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS; + TupleStream solrStream = new SolrStream(url, paramsLoc); + + StreamContext context = new StreamContext(); + solrStream.setStreamContext(context); + List tuples = getTuples(solrStream); + assertTrue(tuples.size() == 1); + Number number = (Number)tuples.get(0).get("return-value"); + assertTrue(number.doubleValue() == .5D); + } From fc7e2137431309b599419042eeb50b74c9b9e8e3 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Fri, 11 Aug 2017 16:18:53 +0930 Subject: [PATCH 07/11] added a specic nopte that schema.xml components canot be loaded from blob store --- solr/solr-ref-guide/src/blob-store-api.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/solr/solr-ref-guide/src/blob-store-api.adoc b/solr/solr-ref-guide/src/blob-store-api.adoc index 267ed1db1ab..668acef0cb0 100644 --- a/solr/solr-ref-guide/src/blob-store-api.adoc +++ b/solr/solr-ref-guide/src/blob-store-api.adoc @@ -146,3 +146,8 @@ For example, to use a blob named test, you would configure `solrconfig.xml` like ---- If there are parameters available in the custom handler, you can define them in the same way as any other request handler definition. + +[NOTE] +==== + +Blob store can only be used to dynamically load components configured in `solrconfig.xml`. Components specified in `schema.xml` cannot be loaded from blob store \ No newline at end of file From 6e9538c719d4a956ba10cec4590cdedc13f4d162 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Fri, 11 Aug 2017 18:35:03 +0530 Subject: [PATCH 08/11] Fix ignored test --- .../hadoop/TestZkAclsWithHadoopAuth.java | 127 +++++------------- 1 file changed, 37 insertions(+), 90 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java index a597a2fe945..3bc1c0bc309 100644 --- a/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java +++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java @@ -16,10 +16,13 @@ */ package org.apache.solr.security.hadoop; +import static org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME; +import static org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME; +import static org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME; +import static org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME; + import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; +import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Locale; import java.util.concurrent.TimeUnit; @@ -28,10 +31,10 @@ import org.apache.lucene.util.Constants; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.cloud.ZkCredentialsProvider; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider; +import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.Code; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs; @@ -39,30 +42,33 @@ import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; import org.apache.zookeeper.data.Stat; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.auth.AuthenticationProvider; -import org.apache.zookeeper.server.auth.ProviderRegistry; +import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; -@Ignore public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { protected static final int NUM_SERVERS = 1; protected static final int NUM_SHARDS = 1; protected static final int REPLICATION_FACTOR = 1; + private static final String SOLR_PASSWD = "solr"; + private static final String FOO_PASSWD = "foo"; + private static final Id SOLR_ZK_ID = new Id("digest", digest ("solr", SOLR_PASSWD)); + private static final Id FOO_ZK_ID = new Id("digest", digest ("foo", FOO_PASSWD)); @BeforeClass public static void setupClass() throws Exception { assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS); assumeFalse("FIXME: SOLR-8182: This test fails under Java 9", Constants.JRE_IS_MINIMUM_JAVA9); - System.setProperty("zookeeper.authProvider.1", DummyZKAuthProvider.class.getName()); - System.setProperty("zkCredentialsProvider", DummyZkCredentialsProvider.class.getName()); - System.setProperty("zkACLProvider", DummyZkAclProvider.class.getName()); - - ProviderRegistry.initialize(); + System.setProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + VMParamsAllAndReadonlyDigestZkACLProvider.class.getName()); + System.setProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName()); + System.setProperty(DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "solr"); + System.setProperty(DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, SOLR_PASSWD); + System.setProperty(DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, "foo"); + System.setProperty(DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME, FOO_PASSWD); configureCluster(NUM_SERVERS)// nodes .withSolrXml(MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML) @@ -73,9 +79,12 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { @AfterClass public static void tearDownClass() { - System.clearProperty("zookeeper.authProvider.1"); - System.clearProperty("zkCredentialsProvider"); - System.clearProperty("zkACLProvider"); + System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME); + System.clearProperty(DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME); + System.clearProperty(DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME); + System.clearProperty(DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME); } @Test @@ -89,7 +98,7 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { } }); - keeper.addAuthInfo("dummyauth", "solr".getBytes(StandardCharsets.UTF_8)); + keeper.addAuthInfo("digest", ("solr:"+SOLR_PASSWD).getBytes(StandardCharsets.UTF_8)); // Test well known paths. checkNonSecurityACLs(keeper, "/solr.xml"); @@ -135,84 +144,22 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { List acls = keeper.getACL(path, new Stat()); String message = String.format(Locale.ROOT, "Path %s ACLs found %s", path, acls); assertEquals(message, 1, acls.size()); - assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.ALL, new Id("dummyauth", "solr")))); + assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.ALL, SOLR_ZK_ID))); } private void checkNonSecurityACLs(ZooKeeper keeper, String path) throws Exception { List acls = keeper.getACL(path, new Stat()); String message = String.format(Locale.ROOT, "Path %s ACLs found %s", path, acls); assertEquals(message, 2, acls.size()); - assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.ALL, new Id("dummyauth", "solr")))); - assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.READ, new Id("world", "anyone")))); + assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.ALL, SOLR_ZK_ID))); + assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.READ, FOO_ZK_ID))); } - public static class DummyZKAuthProvider implements AuthenticationProvider { - public static final String zkSuperUser = "zookeeper"; - public static final Collection validUsers = Arrays.asList(zkSuperUser, "solr", "foo"); - - @Override - public String getScheme() { - return "dummyauth"; - } - - @Override - public Code handleAuthentication(ServerCnxn arg0, byte[] arg1) { - String userName = new String(arg1, StandardCharsets.UTF_8); - - if (validUsers.contains(userName)) { - if (zkSuperUser.equals(userName)) { - arg0.addAuthInfo(new Id("super", "")); - } - arg0.addAuthInfo(new Id(getScheme(), userName)); - return KeeperException.Code.OK; - } - - return KeeperException.Code.AUTHFAILED; - } - - @Override - public boolean isAuthenticated() { - return true; - } - - @Override - public boolean isValid(String arg0) { - return (arg0 != null) && validUsers.contains(arg0); - } - - @Override - public boolean matches(String arg0, String arg1) { - return arg0.equals(arg1); + private static String digest (String userName, String passwd) { + try { + return DigestAuthenticationProvider.generateDigest(userName+":"+passwd); + } catch (NoSuchAlgorithmException ex) { + throw new RuntimeException(ex); } } - - public static class DummyZkCredentialsProvider implements ZkCredentialsProvider { - public static final Collection solrCreds = - Arrays.asList(new ZkCredentials("dummyauth", "solr".getBytes(StandardCharsets.UTF_8))); - - @Override - public Collection getCredentials() { - return solrCreds; - } - } - - public static class DummyZkAclProvider extends SecurityAwareZkACLProvider { - - @Override - protected List createNonSecurityACLsToAdd() { - List result = new ArrayList<>(2); - result.add(new ACL(ZooDefs.Perms.ALL, new Id("dummyauth", "solr"))); - result.add(new ACL(ZooDefs.Perms.READ, ZooDefs.Ids.ANYONE_ID_UNSAFE)); - - return result; - } - - @Override - protected List createSecurityACLsToAdd() { - List ret = new ArrayList(); - ret.add(new ACL(ZooDefs.Perms.ALL, new Id("dummyauth", "solr"))); - return ret; - } - } - } From 3959c53eafc9e5cc55c48cbf6a5382b0f9bffa75 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Fri, 11 Aug 2017 12:34:52 +0100 Subject: [PATCH 09/11] SOLR-11195: Require class attribute for shard and cluster metric reporter configuration. --- solr/CHANGES.txt | 8 +++++++- .../apache/solr/metrics/SolrMetricManager.java | 18 ++++++------------ .../src/test-files/solr/solr-solrreporter.xml | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5cfd0ea41cf..86eb24721c4 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -52,7 +52,11 @@ Detailed Change List Upgrade Notes ---------------------- -(No Notes) +* SOLR-11195: shard and cluster metric reporter configuration now requires a class attribute. + If a reporter configures the group="shard" attribute then please also configure the + class="org.apache.solr.metrics.reporters.solr.SolrShardReporter" attribute. + If a reporter configures the group="cluster" attribute then please also configure the + class="org.apache.solr.metrics.reporters.solr.SolrClusterReporter" attribute. New Features ---------------------- @@ -141,6 +145,8 @@ Other Changes * SOLR-11071: Improve TestIntervalFacets.testRandom (Tomás Fernández Löbbe) +* SOLR-11195: Require class attribute for shard and cluster metric reporter configuration. (Christine Poerschke) + ================== 7.0.0 ================== Versions of Major Components diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java index 8b17c78fe0b..001717c3844 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java @@ -1003,7 +1003,7 @@ public class SolrMetricManager { } } - private List prepareCloudPlugins(PluginInfo[] pluginInfos, String group, String className, + private List prepareCloudPlugins(PluginInfo[] pluginInfos, String group, Map defaultAttributes, Map defaultInitArgs) { List result = new ArrayList<>(); @@ -1015,7 +1015,7 @@ public class SolrMetricManager { if (!group.equals(groupAttr)) { continue; } - info = preparePlugin(info, className, defaultAttributes, defaultInitArgs); + info = preparePlugin(info, defaultAttributes, defaultInitArgs); if (info != null) { result.add(info); } @@ -1023,18 +1023,12 @@ public class SolrMetricManager { return result; } - private PluginInfo preparePlugin(PluginInfo info, String className, Map defaultAttributes, + private PluginInfo preparePlugin(PluginInfo info, Map defaultAttributes, Map defaultInitArgs) { if (info == null) { return null; } String classNameAttr = info.attributes.get("class"); - if (className != null) { - if (classNameAttr != null && !className.equals(classNameAttr)) { - log.warn("Conflicting class name attributes, expected " + className + " but was " + classNameAttr + ", skipping " + info); - return null; - } - } Map attrs = new HashMap<>(info.attributes); defaultAttributes.forEach((k, v) -> { @@ -1042,7 +1036,7 @@ public class SolrMetricManager { attrs.put(k, v); } }); - attrs.put("class", className); + attrs.put("class", classNameAttr); Map initArgs = new HashMap<>(); if (info.initArgs != null) { initArgs.putAll(info.initArgs.asMap(10)); @@ -1069,7 +1063,7 @@ public class SolrMetricManager { String registryName = core.getCoreMetricManager().getRegistryName(); // collect infos and normalize - List infos = prepareCloudPlugins(pluginInfos, SolrInfoBean.Group.shard.toString(), SolrShardReporter.class.getName(), + List infos = prepareCloudPlugins(pluginInfos, SolrInfoBean.Group.shard.toString(), attrs, initArgs); for (PluginInfo info : infos) { try { @@ -1092,7 +1086,7 @@ public class SolrMetricManager { attrs.put("group", SolrInfoBean.Group.cluster.toString()); Map initArgs = new HashMap<>(); initArgs.put("period", DEFAULT_CLOUD_REPORTER_PERIOD); - List infos = prepareCloudPlugins(pluginInfos, SolrInfoBean.Group.cluster.toString(), SolrClusterReporter.class.getName(), + List infos = prepareCloudPlugins(pluginInfos, SolrInfoBean.Group.cluster.toString(), attrs, initArgs); String registryName = getRegistryName(SolrInfoBean.Group.cluster); for (PluginInfo info : infos) { diff --git a/solr/core/src/test-files/solr/solr-solrreporter.xml b/solr/core/src/test-files/solr/solr-solrreporter.xml index a66d9d096e1..21f8ac1833c 100644 --- a/solr/core/src/test-files/solr/solr-solrreporter.xml +++ b/solr/core/src/test-files/solr/solr-solrreporter.xml @@ -42,12 +42,12 @@ false - + 5 UPDATE\./update/.*requests QUERY\./select.*requests - + /admin/metrics/collector 5 From 67de1a3f70ad04a8f040dddfa80ccd6ac1277f15 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Fri, 11 Aug 2017 19:48:35 +0530 Subject: [PATCH 10/11] SOLR-9735: Replace nocommit with link to ref guide document --- solr/solrj/src/resources/apispec/autoscaling.Commands.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solrj/src/resources/apispec/autoscaling.Commands.json b/solr/solrj/src/resources/apispec/autoscaling.Commands.json index 5ff89ef89da..93adc5eaa4e 100644 --- a/solr/solrj/src/resources/apispec/autoscaling.Commands.json +++ b/solr/solrj/src/resources/apispec/autoscaling.Commands.json @@ -1,5 +1,5 @@ { - "documentation": "TODO NOCOMMIT", + "documentation": "https://lucene.apache.org/solr/guide/solrcloud-autoscaling-api.html", "description": "The Scaling API provides API for adding cluster level scaling rules, triggers and event listeners", "methods": [ "GET", From bbc368d0efbbf2eb6aa1c3dcc6137e786cdbe5f9 Mon Sep 17 00:00:00 2001 From: Varun Thacker Date: Fri, 11 Aug 2017 11:57:28 -0700 Subject: [PATCH 11/11] SOLR-11228: Exclude static html files in the partials directory from authentication and authorization checks --- solr/CHANGES.txt | 3 +++ solr/webapp/web/WEB-INF/web.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 86eb24721c4..a7b394efaf2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -95,6 +95,9 @@ Bug Fixes * SOLR-11190: GraphQuery also supports string fields which are indexed=false and docValues=true. Please refer to the Javadocs for DocValuesTermsQuery for it's performance characteristics. (Karthik Ramachandran, Varun Thacker) +* SOLR-11228: Exclude static html files in the partials directory from authentication and authorization checks. The UI + will open correctly with kerberos enabled (Ishan Chattopadhyaya, Varun Thacker) + Optimizations ---------------------- diff --git a/solr/webapp/web/WEB-INF/web.xml b/solr/webapp/web/WEB-INF/web.xml index a4dc2338583..8d862696597 100644 --- a/solr/webapp/web/WEB-INF/web.xml +++ b/solr/webapp/web/WEB-INF/web.xml @@ -56,7 +56,7 @@ --> excludePatterns - /libs/.+,/css/.+,/js/.+,/img/.+,/tpl/.+ + /partials/.+,/libs/.+,/css/.+,/js/.+,/img/.+,/tpl/.+