Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9107: new @RandomizeSSL annotation for more fine grained control of SSL testing

This commit is contained in:
Chris Hostetter 2016-06-01 14:34:31 -07:00
parent 7de3ef3510
commit 09372acb66
4 changed files with 390 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -347,6 +347,8 @@ Other Changes
* SOLR-9136: Separate out the error statistics into server-side error vs client-side error
(Jessica Cheng Mallet via Erick Erickson)
* SOLR-9107: new @RandomizeSSL annotation for more fine grained control of SSL testing (hossman, sarowe)
================== 6.0.1 ==================
(No Changes)

202
solr/core/src/test/org/apache/solr/cloud/TestSSLRandomization.java
View File

@ -17,9 +17,13 @@
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.util.SSLTestConfig;
import org.apache.solr.util.RandomizeSSL;
import org.apache.solr.util.RandomizeSSL.SSLRandomizer;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@ -34,6 +38,7 @@ import org.slf4j.LoggerFactory;
*
* @see TestMiniSolrCloudClusterSSL
*/
@RandomizeSSL(ssl=0.5,reason="frequent SSL usage to make test worth while")
public class TestSSLRandomization extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -51,4 +56,201 @@ public class TestSSLRandomization extends SolrCloudTestCase {
String url = buildUrl(6666, "/foo");
assertEquals(sslConfig.isSSLMode() ? "https://127.0.0.1:6666/foo" : "http://127.0.0.1:6666/foo", url);
}
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42,clientAuth=0.33,reason="foo")
public class FullyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedFullyAnnotated extends FullyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class NotAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedNotAnnotated extends NotAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@SuppressSSL(bugUrl="fakeBugUrl")
public class Suppressed { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedSuppressed extends Suppressed { };
/** Used by {@link #testSSLRandomizer} */
@SuppressSSL(bugUrl="fakeBugUrl")
public class InheritedAnnotationButSuppressed extends FullyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42,clientAuth=0.33,reason="foo")
public class InheritedSuppressedWithIgnoredAnnotation extends Suppressed {
// Even with direct annotation, supression at superclass overrules us.
//
// (If it didn't work this way, it would be a pain in the ass to quickly disable SSL for a
// broad hierarchy of tests)
};
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL()
public class EmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedEmptyAnnotated extends EmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(0.5)
public class InheritedEmptyAnnotatationWithOverride extends EmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42,clientAuth=0.33,reason="foo")
public class GrandchildInheritedEmptyAnnotatationWithOverride extends InheritedEmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(0.5)
public class SimplyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(0.0)
public class MinAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(1)
public class MaxAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42)
public class SSlButNoClientAuthAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(clientAuth=0.42)
public class ClientAuthButNoSSLAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=42.0)
public class SSLOutOfRangeAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(clientAuth=42.0)
public class ClientAuthOutOfRangeAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedOutOfRangeAnnotated extends ClientAuthOutOfRangeAnnotated { };
public void testSSLRandomizer() {
SSLRandomizer r;
// for some cases, we know exactly what the config should be regardless of randomization factors
SSLTestConfig conf;
for (Class c : Arrays.asList(FullyAnnotated.class, InheritedFullyAnnotated.class,
GrandchildInheritedEmptyAnnotatationWithOverride.class )) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), 0.42D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.33D, r.clientAuth, 0.0D);
assertTrue(c.toString(), r.debug.contains("foo"));
}
for (Class c : Arrays.asList(NotAnnotated.class, InheritedNotAnnotated.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), 0.0D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.0D, r.clientAuth, 0.0D);
assertTrue(c.toString(), r.debug.contains("not specified"));
conf = r.createSSLTestConfig();
assertEquals(c.toString(), false, conf.isSSLMode());
assertEquals(c.toString(), false, conf.isClientAuthMode());
}
for (Class c : Arrays.asList(Suppressed.class,
InheritedSuppressed.class,
InheritedAnnotationButSuppressed.class,
InheritedSuppressedWithIgnoredAnnotation.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(Suppressed.class);
assertEquals(c.toString(), 0.0D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.0D, r.clientAuth, 0.0D);
assertTrue(c.toString(), r.debug.contains("SuppressSSL"));
assertTrue(c.toString(), r.debug.contains("fakeBugUrl"));
conf = r.createSSLTestConfig();
assertEquals(c.toString(), false, conf.isSSLMode());
assertEquals(c.toString(), false, conf.isClientAuthMode());
}
for (Class c : Arrays.asList(EmptyAnnotated.class, InheritedEmptyAnnotated.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), RandomizeSSL.DEFAULT_ODDS, r.ssl, 0.0D);
assertEquals(c.toString(), RandomizeSSL.DEFAULT_ODDS, r.clientAuth, 0.0D);
}
for (Class c : Arrays.asList(SimplyAnnotated.class, InheritedEmptyAnnotatationWithOverride.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), 0.5D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.5D, r.clientAuth, 0.0D);
}
r = SSLRandomizer.getSSLRandomizerForClass(MinAnnotated.class);
assertEquals(0.0D, r.ssl, 0.0D);
assertEquals(0.0D, r.clientAuth, 0.0D);
conf = r.createSSLTestConfig();
assertEquals(false, conf.isSSLMode());
assertEquals(false, conf.isClientAuthMode());
r = SSLRandomizer.getSSLRandomizerForClass(MaxAnnotated.class);
assertEquals(1.0D, r.ssl, 0.0D);
assertEquals(1.0D, r.clientAuth, 0.0D);
conf = r.createSSLTestConfig();
assertEquals(true, conf.isSSLMode());
assertEquals(true, conf.isClientAuthMode());
r = SSLRandomizer.getSSLRandomizerForClass(SSlButNoClientAuthAnnotated.class);
assertEquals(0.42D, r.ssl, 0.0D);
assertEquals(0.42D, r.clientAuth, 0.0D);
r = SSLRandomizer.getSSLRandomizerForClass(ClientAuthButNoSSLAnnotated.class);
assertEquals(RandomizeSSL.DEFAULT_ODDS, r.ssl, 0.0D);
assertEquals(0.42D, r.clientAuth, 0.0D);
for (Class c : Arrays.asList(SSLOutOfRangeAnnotated.class,
ClientAuthOutOfRangeAnnotated.class,
InheritedOutOfRangeAnnotated.class)) {
expectThrows(IllegalArgumentException.class, () -> {
Object trash = SSLRandomizer.getSSLRandomizerForClass(c);
});
}
}
public void testSSLRandomizerEffectiveOdds() {
assertEquals(RandomizeSSL.DEFAULT_ODDS,
SSLRandomizer.getEffectiveOdds(RandomizeSSL.DEFAULT_ODDS, false, 1), 0.0005D);
assertEquals(0.2727D,
SSLRandomizer.getEffectiveOdds(RandomizeSSL.DEFAULT_ODDS, true, 1), 0.0005D);
assertEquals(0.0100D, SSLRandomizer.getEffectiveOdds(0.01D, false, 1), 0.0005D);
assertEquals(0.1000D, SSLRandomizer.getEffectiveOdds(0.01D, true, 1), 0.0005D);
assertEquals(0.6206D, SSLRandomizer.getEffectiveOdds(0.01D, false, 5), 0.0005D);
assertEquals(0.5000D, SSLRandomizer.getEffectiveOdds(0.5D, false, 1), 0.0005D);
assertEquals(0.5454D, SSLRandomizer.getEffectiveOdds(0.5D, true, 1), 0.0005D);
assertEquals(0.8083D, SSLRandomizer.getEffectiveOdds(0.5D, false, 5), 0.0005D);
assertEquals(0.8000D, SSLRandomizer.getEffectiveOdds(0.8D, false, 1), 0.0005D);
assertEquals(0.8181D, SSLRandomizer.getEffectiveOdds(0.8D, true, 1), 0.0005D);
assertEquals(0.9233D, SSLRandomizer.getEffectiveOdds(0.8D, false, 5), 0.0005D);
// never ever
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, false, 1), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, true, 100), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, false, 100), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, true, 10000), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, false, 10000), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, random().nextBoolean(), random().nextInt()), 0.0D);
// always
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, false, 1), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, true, 100), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, false, 100), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, true, 10000), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, false, 10000), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, random().nextBoolean(), random().nextInt()), 0.0D);
}
}

27
solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
View File

@ -104,6 +104,8 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.servlet.DirectSolrConnection;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.RandomizeSSL;
import org.apache.solr.util.RandomizeSSL.SSLRandomizer;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.RevertDefaultThreadHandlerRule;
import org.apache.solr.util.SSLTestConfig;
@ -137,6 +139,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
})
@SuppressSysoutChecks(bugUrl = "Solr dumps tons of logs to console.")
@SuppressFileSystems("ExtrasFS") // might be ok, the failures with e.g. nightly runs might be "normal"
@RandomizeSSL()
public abstract class SolrTestCaseJ4 extends LuceneTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -317,27 +320,21 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
}
private static SSLTestConfig buildSSLConfig() {
// test has been disabled
if (RandomizedContext.current().getTargetClass().isAnnotationPresent(SuppressSSL.class)) {
return new SSLTestConfig();
}
SSLRandomizer sslRandomizer =
SSLRandomizer.getSSLRandomizerForClass(RandomizedContext.current().getTargetClass());
// we don't choose ssl that often because of SOLR-5776
final boolean trySsl = random().nextInt(10) < 2;
// NOTE: clientAuth is useless unless trySsl==true, but we randomize it independently
// just in case it might find bugs in our test/ssl client code (ie: attempting to use
// SSL w/client cert to non-ssl servers)
boolean trySslClientAuth = random().nextInt(10) < 2;
if (Constants.MAC_OS_X) {
// see SOLR-9039
// If a solution is found to remove this, please make sure to also update
// TestMiniSolrCloudClusterSSL.testSslAndClientAuth as well.
trySslClientAuth = false;
sslRandomizer = new SSLRandomizer(sslRandomizer.ssl, 0.0D, (sslRandomizer.debug + " w/ MAC_OS_X supressed clientAuth"));
}
log.info("Randomized ssl ({}) and clientAuth ({})", trySsl, trySslClientAuth);
return new SSLTestConfig(trySsl, trySslClientAuth);
SSLTestConfig result = sslRandomizer.createSSLTestConfig();
log.info("Randomized ssl ({}) and clientAuth ({}) via: {}",
result.isSSLMode(), result.isClientAuthMode(), sslRandomizer.debug);
return result;
}
protected static JettyConfig buildJettyConfig(String context) {

174
solr/test-framework/src/java/org/apache/solr/util/RandomizeSSL.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.util;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
/**
* Marker annotation indicating when SSL Randomization should be used for a test class, and if so what
* the typical odds of using SSL should for that test class.
* @see SSLRandomizer#getSSLRandomizerForClass
* @see SuppressSSL
*/
@Documented
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface RandomizeSSL {
// we don't choose ssl that often by default because of SOLR-5776
public static final double DEFAULT_ODDS = 0.2D;
/** Comment to inlcude when logging details of SSL randomization */
public String reason() default "";
/**
* Odds (as ratio relative to 1) that SSL should be selected in a typical run.
* Must either be betwen 0.0 and 1.0 (inclusively) or NaN in which case a sensible should be used.
* Actual Odds used for randomization may be higher depending on runner options such as
* <code>tests.multiplier</code> or <code>tests.nightly</code>
*
* @see #DEFAULT_ODDS
* @see LuceneTestCase#TEST_NIGHTLY
* @see LuceneTestCase#RANDOM_MULTIPLIER
*/
public double ssl() default Double.NaN;
/**
* Odds (as ratio relative to 1) that SSL should be selected in a typical run.
* Must either be betwen 0.0 and 1.0 (inclusively) or NaN in which case the effective value of
* {@link #ssl} should be used.
* Actual Odds used for randomization may be higher depending on runner options such as
* <code>tests.multiplier</code> or <code>tests.nightly</code>
* <p>
* NOTE: clientAuth is useless unless ssl is also in used, but we randomize it independently
* just in case it might find bugs in our test/ssl client code (ie: attempting to use
* SSL w/client cert to non-ssl servers)
* </p>
* @see #DEFAULT_ODDS
* @see LuceneTestCase#TEST_NIGHTLY
* @see LuceneTestCase#RANDOM_MULTIPLIER
*/
public double clientAuth() default Double.NaN;
/**
* A shorthand option for controlling both {@link #ssl} and {@link #clientAuth} with a single numeric
* value, For example: <code>@RandomizeSSL(0.5)</code>.
*
* Ignored if {@link #ssl} is set explicitly.
*/
public double value() default Double.NaN;
/**
* A simple data structure for encapsulating the effective values to be used when randomizing
* SSL in a test, based on the configured values in the {@link RandomizeSSL} annotation.
*/
public static final class SSLRandomizer {
public final double ssl;
public final double clientAuth;
public final String debug;
/** @lucene.internal */
public SSLRandomizer(double ssl, double clientAuth, String debug) {
this.ssl = ssl;
this.clientAuth = clientAuth;
this.debug = debug;
}
/**
* Randomly produces an SSLTestConfig taking into account various factors
*
* @see LuceneTestCase#TEST_NIGHTLY
* @see LuceneTestCase#RANDOM_MULTIPLIER
* @see LuceneTestCase#random()
*/
public SSLTestConfig createSSLTestConfig() {
// even if we know SSL is disabled, always consume the same amount of randomness
// that way all other test behavior should be consistent even if a user adds/removes @SuppressSSL
final boolean useSSL = TestUtil.nextInt(LuceneTestCase.random(), 0, 1000) <
(int)(1000 * getEffectiveOdds(ssl, LuceneTestCase.TEST_NIGHTLY, LuceneTestCase.RANDOM_MULTIPLIER));
final boolean useClientAuth = TestUtil.nextInt(LuceneTestCase.random(), 0, 1000) <
(int)(1000 * getEffectiveOdds(clientAuth, LuceneTestCase.TEST_NIGHTLY, LuceneTestCase.RANDOM_MULTIPLIER));
return new SSLTestConfig(useSSL, useClientAuth);
}
/** @lucene.internal Public only for testing */
public static double getEffectiveOdds(final double declaredOdds,
final boolean nightly,
final int multiplier) {
assert declaredOdds <= 1.0D;
assert 0.0D <= declaredOdds;
if (declaredOdds == 0.0D || declaredOdds == 1.0D ) {
return declaredOdds;
}
assert 0 < multiplier;
// negate the odds so we can then divide it by our multipling factors
// to increase the final odds
return 1.0D - ((1.0D - declaredOdds)
/ ((nightly ? 1.1D : 1.0D) * (1.0D + Math.log(multiplier))));
}
/**
* Returns an SSLRandomizer suitable for the specified (test) class
*/
public static final SSLRandomizer getSSLRandomizerForClass(Class clazz) {
final SuppressSSL suppression = (SuppressSSL) clazz.getAnnotation(SuppressSSL.class);
if (null != suppression) {
// Even if this class has a RandomizeSSL annotation, any usage of SuppressSSL -- even in a
// super class -- overrules that.
//
// (If it didn't work this way, it would be a pain in the ass to quickly disable SSL for a
// broad hierarchy of tests)
return new SSLRandomizer(0.0D, 0.0D, suppression.toString());
}
final RandomizeSSL annotation = (RandomizeSSL) clazz.getAnnotation(RandomizeSSL.class);
if (null == annotation) {
return new SSLRandomizer(0.0D, 0.0D, RandomizeSSL.class.getName() + " annotation not specified");
}
final double def = Double.isNaN(annotation.value()) ? DEFAULT_ODDS : annotation.value();
if (def < 0.0D || 1.0D < def) {
throw new IllegalArgumentException
(clazz.getName() + ": default value is not a ratio between 0 and 1: " + annotation.toString());
}
final double ssl = Double.isNaN(annotation.ssl()) ? def : annotation.ssl();
if (ssl < 0.0D || 1.0D < ssl) {
throw new IllegalArgumentException
(clazz.getName() + ": ssl value is not a ratio between 0 and 1: " + annotation.toString());
}
final double clientAuth = Double.isNaN(annotation.clientAuth()) ? ssl : annotation.clientAuth();
if (clientAuth < 0.0D || 1 < clientAuth) {
throw new IllegalArgumentException
(clazz.getName() + ": clientAuth value is not a ratio between 0 and 1: " + annotation.toString());
}
return new SSLRandomizer(ssl, clientAuth, annotation.toString());
}
}
}