run tests with AssertingCodec to find bugs

dev-tools/forbidden/test-signatures.txt

@@ -16,3 +16,5 @@
 
 com.carrotsearch.randomizedtesting.RandomizedTest#globalTempDir() @ Use newTempDirPath() instead
 com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded seeds
+
+org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead

src/main/java/org/elasticsearch/index/codec/CodecService.java

@@ -50,6 +50,8 @@ public class CodecService extends AbstractIndexComponent {
 
     public final static String DEFAULT_CODEC = "default";
     public final static String BEST_COMPRESSION_CODEC = "best_compression";
+    /** the raw unfiltered lucene default. useful for testing */
+    public final static String LUCENE_DEFAULT_CODEC = "lucene_default";
 
     public CodecService(Index index) {
         this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
@@ -73,6 +75,7 @@ public class CodecService extends AbstractIndexComponent {
             codecs.put(BEST_COMPRESSION_CODEC, 
                     new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
         }
+        codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {
             codecs.put(codec, Codec.forName(codec));
         }

src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java

@@ -875,7 +875,7 @@ public class GetTermVectorsTests extends AbstractTermVectorsTests {
         checkBrownFoxTermVector(resp.getFields(), "field1", false);
     }
 
-    @Test
+    @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/10660")
     public void testArtificialNonExistingField() throws Exception {
         // setup indices
         ImmutableSettings.Builder settings = settingsBuilder()

src/test/java/org/elasticsearch/bwcompat/StaticIndexBackwardCompatibilityTest.java

@@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
  * These tests are against static indexes, built from versions of ES that cannot be upgraded without
  * a full cluster restart (ie no wire format compatibility).
  */
-@LuceneTestCase.SuppressCodecs({"Lucene3x", "MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene40", "Lucene41", "Appending", "Lucene42", "Lucene45", "Lucene46", "Lucene49"})
+@LuceneTestCase.SuppressCodecs("*")
 @ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.TEST, numDataNodes = 0, minNumDataNodes = 0, maxNumDataNodes = 0)
 public class StaticIndexBackwardCompatibilityTest extends ElasticsearchIntegrationTest {
 

src/test/java/org/elasticsearch/index/codec/CodecTests.java

@@ -36,25 +36,18 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.test.ElasticsearchSingleNodeTest;
-import org.junit.Before;
 import org.junit.Test;
 
 import static org.hamcrest.Matchers.instanceOf;
 
+@SuppressCodecs("*") // we test against default codec so never get a random one here!
 public class CodecTests extends ElasticsearchSingleNodeTest {
 
-    @Override
-    @Before
-    public void setUp() throws Exception {
-        super.setUp();
-        Codec.setDefault(TestUtil.getDefaultCodec()); // we test against default codec so never get a random one here!
-    }
-
     @Test
     public void testResolveDefaultCodecs() throws Exception {
         CodecService codecService = createCodecService();

src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTests.java

@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.*;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.ContentPath;
@@ -41,9 +40,6 @@ import org.junit.Before;
 
 import static org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 
-// we might wanna cut this over to LuceneTestCase
-@SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45", "Lucene46"}) 
-// avoid codecs that do not support SortedNumerics, SortedSet, etc
 public abstract class AbstractFieldDataTests extends ElasticsearchSingleNodeTest {
 
     protected IndexService indexService;

src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIntegrationTest.java

@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
 
 import com.google.common.collect.ImmutableMap;
 
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.suggest.SuggestResponse;
@@ -51,6 +52,7 @@ import static org.hamcrest.Matchers.not;
 /**
  * Tests for transforming the source document before indexing.
  */
+@SuppressCodecs("*") // requires custom completion format
 public class TransformOnIndexMapperIntegrationTest extends ElasticsearchIntegrationTest {
     @Test
     public void searchOnTransformed() throws Exception {

src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.indices.stats;
 
 import org.apache.lucene.util.Version;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.*;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
@@ -60,6 +61,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.*;
 
 @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, randomDynamicTemplates = false)
+@SuppressCodecs("*") // requires custom completion format
 public class IndexStatsTests extends ElasticsearchIntegrationTest {
 
     @Override

src/test/java/org/elasticsearch/nested/SimpleNestedTests.java

@@ -164,7 +164,7 @@ public class SimpleNestedTests extends ElasticsearchIntegrationTest {
         assertThat(searchResponse.getHits().totalHits(), equalTo(1l));
     }
 
-    @Test
+    @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/10661")
     public void simpleNestedMatchQueries() throws Exception {
         XContentBuilder builder = jsonBuilder().startObject()
                 .startObject("type1")

src/test/java/org/elasticsearch/recovery/TruncatedRecoveryTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.recovery;
 
 import org.apache.lucene.util.English;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -52,6 +53,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 @ElasticsearchIntegrationTest.ClusterScope(numDataNodes = 2, numClientNodes = 0, scope = ElasticsearchIntegrationTest.Scope.TEST)
+@SuppressCodecs("*") // test relies on exact file extensions
 public class TruncatedRecoveryTests extends ElasticsearchIntegrationTest {
 
     protected Settings nodeSettings(int nodeOrdinal) {

src/test/java/org/elasticsearch/search/aggregations/bucket/TopHitsTests.java

@@ -776,7 +776,7 @@ public class TopHitsTests extends ElasticsearchIntegrationTest {
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0));
     }
 
-    @Test
+    @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/10661")
     public void testNestedFetchFeatures() {
         String hlType = randomFrom("plain", "fvh", "postings");
         HighlightBuilder.Field hlField = new HighlightBuilder.Field("comments.message")

src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchTests.java

@@ -22,6 +22,7 @@ import com.carrotsearch.hppc.ObjectLongOpenHashMap;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.google.common.collect.Lists;
 
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
@@ -68,6 +69,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
 import static org.hamcrest.Matchers.*;
 
+@SuppressCodecs("*") // requires custom completion format
 public class CompletionSuggestSearchTests extends ElasticsearchIntegrationTest {
 
     private final String INDEX = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT);

src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearchTests.java

@@ -41,6 +41,7 @@ import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuil
 import org.elasticsearch.search.suggest.context.ContextBuilder;
 import org.elasticsearch.search.suggest.context.ContextMapping;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.hamcrest.Matchers;
 import org.junit.Test;
 
@@ -52,6 +53,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
 import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertDistance;
 import static org.hamcrest.Matchers.containsString;
 
+@SuppressCodecs("*") // requires custom completion format
 public class ContextSuggestSearchTests extends ElasticsearchIntegrationTest {
 
     private static final String INDEX = "test";

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -28,6 +28,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.google.common.base.Joiner;
 import com.google.common.base.Predicate;
 import com.google.common.collect.Lists;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.lucene.store.StoreRateLimiting;
@@ -95,6 +96,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -369,7 +371,14 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
             randomSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, numberOfShards())
                     .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas());
 
-            randomSettingsBuilder.put("index.codec", randomFrom("default", "best_compression"));
+            // if the test class is annotated with SuppressCodecs("*"), it means don't use lucene's codec randomization
+            // otherwise, use it, it has assertions and so on that can find bugs.
+            SuppressCodecs annotation = getClass().getAnnotation(SuppressCodecs.class);
+            if (annotation != null && annotation.value().length == 1 && "*".equals(annotation.value()[0])) {
+                randomSettingsBuilder.put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC));
+            } else {
+                randomSettingsBuilder.put("index.codec", CodecService.LUCENE_DEFAULT_CODEC);
+            }
             XContentBuilder mappings = null;
             if (frequently() && randomDynamicTemplates()) {
                 mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_");

src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java

@@ -93,7 +93,13 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllS
 @ThreadLeakLingering(linger = 5000) // 5 sec lingering
 @TimeoutSuite(millis = 20 * TimeUnits.MINUTE)
 @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
-@SuppressCodecs({"SimpleText", "Memory", "CheapBastard", "Direct"}) // slow ones
+// we suppress pretty much all the lucene codecs for now, except asserting
+// assertingcodec is the winner for a codec here: it finds bugs and gives clear exceptions.
+@SuppressCodecs({
+    "SimpleText", "Memory", "CheapBastard", "Direct", "Compressing", "FST50", "FSTOrd50", 
+    "TestBloomFilteredLucenePostings", "MockRandom", "BlockTreeOrds", "LuceneFixedGap", 
+    "LuceneVarGapFixedInterval", "LuceneVarGapDocFreqInterval", "Lucene50"
+})
 @LuceneTestCase.SuppressReproduceLine
 public abstract class ElasticsearchTestCase extends LuceneTestCase {
     

src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTestCase.java

@@ -27,6 +27,7 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import com.google.common.collect.Lists;
 
 import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.LuceneTestCase.SuppressFsync;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.common.Strings;
@@ -72,6 +73,7 @@ import java.util.Set;
 @ElasticsearchRestTestCase.Rest
 @Slow
 @SuppressFsync // we aren't trying to test this here, and it can make the test slow
+@SuppressCodecs("*") // requires custom completion postings format
 @ClusterScope(randomDynamicTemplates = false)
 @TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // timeout the suite after 40min and fail the test.
 public abstract class ElasticsearchRestTestCase extends ElasticsearchIntegrationTest {