[TEST] add warnings check to ESTestCase

We are currenlty checking that no deprecation warnings are emitted in our query tests. That can be moved to ESTestCase (disabled in ESIntegTestCase) as it allows us to easily catch where our tests use deprecated features and assert on the expected warnings.

core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java

@@ -21,7 +21,6 @@ package org.elasticsearch.common.logging;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.common.SuppressLoggerChecks;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 
@@ -42,7 +41,7 @@ public class DeprecationLogger {
      *
      * https://tools.ietf.org/html/rfc7234#section-5.5
      */
-    public static final String DEPRECATION_HEADER = "Warning";
+    public static final String WARNING_HEADER = "Warning";
 
     /**
      * This is set once by the {@code Node} constructor, but it uses {@link CopyOnWriteArraySet} to ensure that tests can run in parallel.
@@ -128,7 +127,7 @@ public class DeprecationLogger {
 
             while (iterator.hasNext()) {
                 try {
-                    iterator.next().addResponseHeader(DEPRECATION_HEADER, formattedMessage);
+                    iterator.next().addResponseHeader(WARNING_HEADER, formattedMessage);
                 } catch (IllegalStateException e) {
                     // ignored; it should be removed shortly
                 }

core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java

@@ -42,12 +42,14 @@ public class BWCTemplateTests extends ESSingleNodeTestCase {
         client().prepareIndex("packetbeat-foo", "doc", "1").setSource("message", "foo").get();
         client().prepareIndex("filebeat-foo", "doc", "1").setSource("message", "foo").get();
         client().prepareIndex("winlogbeat-foo", "doc", "1").setSource("message", "foo").get();
+        assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
     }
 
     public void testLogstashTemplatesBWC() throws Exception {
         String ls5x = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json");
         client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x).get();
         client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get();
+        assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
     }
 
 }

core/src/test/java/org/elasticsearch/common/ParseFieldTests.java

@@ -20,13 +20,15 @@ package org.elasticsearch.common;
 
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.sameInstance;
 import static org.hamcrest.collection.IsArrayContainingInAnyOrder.arrayContainingInAnyOrder;
 
 public class ParseFieldTests extends ESTestCase {
-    public void testParse() {
+    public void testParse() throws IOException {
         String name = "foo_bar";
         ParseField field = new ParseField(name);
         String[] deprecated = new String[]{"barFoo", "bar_foo", "Foobar"};
@@ -42,33 +44,21 @@ public class ParseFieldTests extends ESTestCase {
         assertThat(withDeprecations.match("foo bar"), is(false));
         for (String deprecatedName : deprecated) {
             assertThat(withDeprecations.match(deprecatedName), is(true));
+            assertWarnings("Deprecated field [" + deprecatedName + "] used, expected [foo_bar] instead");
         }
     }
 
-    public void testAllDeprecated() {
+    public void testAllDeprecated() throws IOException {
         String name = "like_text";
-
-        boolean withDeprecatedNames = randomBoolean();
         String[] deprecated = new String[]{"text", "same_as_text"};
-        String[] allValues;
+        ParseField field = new ParseField(name).withDeprecation(deprecated).withAllDeprecated("like");
-        if (withDeprecatedNames) {
+        assertFalse(field.match("not a field name"));
-            String[] newArray = new String[1 + deprecated.length];
+        assertTrue(field.match("text"));
-            newArray[0] = name;
+        assertWarnings("Deprecated field [text] used, replaced by [like]");
-            System.arraycopy(deprecated, 0, newArray, 1, deprecated.length);
+        assertTrue(field.match("same_as_text"));
-            allValues = newArray;
+        assertWarnings("Deprecated field [same_as_text] used, replaced by [like]");
-        } else {
+        assertTrue(field.match("like_text"));
-            allValues = new String[] {name};
+        assertWarnings("Deprecated field [like_text] used, replaced by [like]");
-        }
-
-        ParseField field;
-        if (withDeprecatedNames) {
-            field = new ParseField(name).withDeprecation(deprecated).withAllDeprecated("like");
-        } else {
-            field = new ParseField(name).withAllDeprecated("like");
-        }
-
-        assertThat(field.match(randomFrom(allValues)), is(true));
-        assertThat(field.match("not a field name"), is(false));
     }
 
     public void testGetAllNamesIncludedDeprecated() {

core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java

@@ -41,6 +41,12 @@ public class DeprecationLoggerTests extends ESTestCase {
 
     private final DeprecationLogger logger = new DeprecationLogger(Loggers.getLogger(getClass()));
 
+    @Override
+    protected boolean enableWarningsCheck() {
+        //this is a low level test for the deprecation logger, setup and checks are done manually
+        return false;
+    }
+
     public void testAddsHeaderWithThreadContext() throws IOException {
         String msg = "A simple message [{}]";
         String param = randomAsciiOfLengthBetween(1, 5);
@@ -54,7 +60,7 @@ public class DeprecationLoggerTests extends ESTestCase {
             Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
 
             assertEquals(1, responseHeaders.size());
-            assertEquals(formatted, responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER).get(0));
+            assertEquals(formatted, responseHeaders.get(DeprecationLogger.WARNING_HEADER).get(0));
         }
     }
 
@@ -74,7 +80,7 @@ public class DeprecationLoggerTests extends ESTestCase {
 
             assertEquals(1, responseHeaders.size());
 
-            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            List<String> responses = responseHeaders.get(DeprecationLogger.WARNING_HEADER);
 
             assertEquals(2, responses.size());
             assertEquals(formatted, responses.get(0));
@@ -93,7 +99,7 @@ public class DeprecationLoggerTests extends ESTestCase {
             logger.deprecated(expected);
 
             Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
-            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            List<String> responses = responseHeaders.get(DeprecationLogger.WARNING_HEADER);
 
             // ensure it works (note: concurrent tests may be adding to it, but in different threads, so it should have no impact)
             assertThat(responses, hasSize(atLeast(1)));
@@ -104,7 +110,7 @@ public class DeprecationLoggerTests extends ESTestCase {
             logger.deprecated(unexpected);
 
             responseHeaders = threadContext.getResponseHeaders();
-            responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            responses = responseHeaders.get(DeprecationLogger.WARNING_HEADER);
 
             assertThat(responses, hasSize(atLeast(1)));
             assertThat(responses, hasItem(expected));

core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java

@@ -22,9 +22,6 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.AbstractObjectParser.NoContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
@@ -38,8 +35,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 
 public class ObjectParserTests extends ESTestCase {
@@ -224,26 +219,16 @@ public class ObjectParserTests extends ESTestCase {
     }
 
     public void testDeprecationWarnings() throws IOException {
-        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+        class TestStruct {
-            DeprecationLogger.setThreadContext(threadContext);
+            public String test;
-            class TestStruct {
-                public String test;
-            }
-            ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
-            TestStruct s = new TestStruct();
-
-            objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
-
-            XContentParser parser = createParser(XContentType.JSON.xContent(), "{\"old_test\" : \"foo\"}");
-            objectParser.parse(parser, s, () -> ParseFieldMatcher.EMPTY);
-
-            assertEquals("foo", s.test);
-
-            final List<String> warnings = threadContext.getResponseHeaders().get(DeprecationLogger.DEPRECATION_HEADER);
-            assertThat(warnings, hasSize(1));
-            assertThat(warnings, hasItem(equalTo("Deprecated field [old_test] used, expected [test] instead")));
-            DeprecationLogger.removeThreadContext(threadContext);
         }
+        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        TestStruct s = new TestStruct();
+        XContentParser parser = createParser(XContentType.JSON.xContent(), "{\"old_test\" : \"foo\"}");
+        objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
+        objectParser.parse(parser, s, () -> ParseFieldMatcher.EMPTY);
+        assertEquals("foo", s.test);
+        assertWarnings("Deprecated field [old_test] used, expected [test] instead");
     }
 
     public void testFailOnValueType() throws IOException {

core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java

@@ -104,7 +104,7 @@ public class AnalysisRegistryTests extends ESTestCase {
         assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
     }
 
-    public void testBackCompatOverrideDefaultIndexAnalyzer() {
+    public void testBackCompatOverrideDefaultIndexAnalyzer() throws IOException {
         Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
                 VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
@@ -113,6 +113,8 @@ public class AnalysisRegistryTests extends ESTestCase {
         assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+        assertWarnings("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] " +
+                "instead for index [index]");
     }
 
     public void testOverrideDefaultSearchAnalyzer() {
@@ -125,7 +127,7 @@ public class AnalysisRegistryTests extends ESTestCase {
         assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
     }
 
-    public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() {
+    public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() throws IOException {
         Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
                 VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
@@ -137,6 +139,8 @@ public class AnalysisRegistryTests extends ESTestCase {
         assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertWarnings("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] " +
+                "instead for index [index]");
     }
 
     public void testConfigureCamelCaseTokenFilter() throws IOException {

core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -43,7 +44,7 @@ public class DynamicTemplateTests extends ESTestCase {
         assertEquals("Illegal dynamic template parameter: [random_param]", e.getMessage());
     }
 
-    public void testParseUnknownMatchType() {
+    public void testParseUnknownMatchType() throws IOException {
         Map<String, Object> templateDef = new HashMap<>();
         templateDef.put("match_mapping_type", "short");
         templateDef.put("mapping", Collections.singletonMap("store", true));

core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java

@@ -160,12 +160,13 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         assertThat(e.getMessage(), containsString("Limit of mapping depth [1] in index [test1] has been exceeded"));
     }
 
-    public void testUnmappedFieldType() {
+    public void testUnmappedFieldType() throws IOException {
         MapperService mapperService = createIndex("index").mapperService();
         assertThat(mapperService.unmappedFieldType("keyword"), instanceOf(KeywordFieldType.class));
         assertThat(mapperService.unmappedFieldType("long"), instanceOf(NumberFieldType.class));
         // back compat
         assertThat(mapperService.unmappedFieldType("string"), instanceOf(KeywordFieldType.class));
+        assertWarnings("[unmapped_type:string] should be replaced with [unmapped_type:keyword]");
     }
 
     public void testMergeWithMap() throws Throwable {
@@ -206,9 +207,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
             .startObject("properties")
             .startObject("field")
             .field("type", "text")
-            .startObject("norms")
+            .field("norms", false)
-            .field("enabled", false)
-            .endObject()
             .endObject()
             .endObject().endObject().bytes());
 

core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java

@@ -422,7 +422,7 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
                 "}";
 
         parseQuery(json);
-        assertWarningHeaders("Deprecated field [coerce] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     public void testFromJsonIgnoreMalformedIsDeprecated() throws IOException {
@@ -440,7 +440,7 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
                 "  }\n" +
                 "}";
         parseQuery(json);
-        assertWarningHeaders("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     @Override

core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java

@@ -330,7 +330,7 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
                 "  }\n" +
                 "}";
         parseQuery(json);
-        assertWarningHeaders("Deprecated field [optimize_bbox] used, replaced by [no replacement: " +
+        assertWarnings("Deprecated field [optimize_bbox] used, replaced by [no replacement: " +
                 "`optimize_bbox` is no longer supported due to recent improvements]");
     }
 
@@ -347,7 +347,7 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
                 "  }\n" +
                 "}";
         parseQuery(json);
-        assertWarningHeaders("Deprecated field [coerce] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     public void testFromJsonIgnoreMalformedIsDeprecated() throws IOException {
@@ -363,7 +363,7 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
                 "  }\n" +
                 "}";
         parseQuery(json);
-        assertWarningHeaders("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     @Override

core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java

@@ -140,7 +140,7 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         builder.endObject();
         builder.endObject();
         parseQuery(builder.string());
-        assertWarningHeaders("Deprecated field [normalize] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [normalize] used, replaced by [validation_method]");
     }
 
     public void testParsingAndToQueryParsingExceptions() throws IOException {
@@ -266,7 +266,7 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
                 "  }\n" +
                 "}";
         parseQuery(json);
-        assertWarningHeaders("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     public void testFromJsonCoerceDeprecated() throws IOException {
@@ -282,7 +282,7 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
                 "  }\n" +
                 "}";
         parseQuery(json);
-        assertWarningHeaders("Deprecated field [coerce] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     @Override

core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java

@@ -23,7 +23,6 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -151,7 +150,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
         builder.endObject();
         HasParentQueryBuilder queryBuilder = (HasParentQueryBuilder) parseQuery(builder.string());
         assertEquals("foo", queryBuilder.type());
-        assertWarningHeaders("Deprecated field [type] used, expected [parent_type] instead");
+        assertWarnings("Deprecated field [type] used, expected [parent_type] instead");
     }
 
     public void testToQueryInnerQueryType() throws IOException {

core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java

@@ -165,7 +165,7 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
         assertEquals(testQuery, parsed);
 
         parseQuery(contentString);
-        assertWarningHeaders("Deprecated field [_type] used, expected [type] instead");
+        assertWarnings("Deprecated field [_type] used, expected [type] instead");
 
         //array of types can also be called types rather than type
         final String contentString2 = "{\n" +
@@ -178,6 +178,6 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
         assertEquals(testQuery, parsed);
 
         parseQuery(contentString2);
-        assertWarningHeaders("Deprecated field [types] used, expected [type] instead");
+        assertWarnings("Deprecated field [types] used, expected [type] instead");
     }
 }

core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java

@@ -318,7 +318,7 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
 
         assertSerialization(qb);
 
-        assertWarningHeaders("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
+        assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
                 "Deprecated field [slop] used, replaced by [match_phrase query]");
     }
 
@@ -347,7 +347,7 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
 
         assertEquals(json, expectedQB, qb);
         assertSerialization(qb);
-        assertWarningHeaders("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
+        assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
                 "Deprecated field [slop] used, replaced by [match_phrase query]");
     }
 

core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java

@@ -387,7 +387,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
                 "}";
 
         assertNotNull(parseQuery(deprecatedJson));
-        assertWarningHeaders("Deprecated field [_name] used, replaced by [query name is not supported in short version of range query]");
+        assertWarnings("Deprecated field [_name] used, replaced by [query name is not supported in short version of range query]");
     }
 
     public void testRewriteDateToMatchAll() throws IOException {

core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java

@@ -136,6 +136,10 @@ public class AnalysisModuleTests extends ModuleTestCase {
         IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);
         assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
         assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(EnglishAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar_search.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasReferencesAlias() throws IOException {
@@ -154,6 +158,10 @@ public class AnalysisModuleTests extends ModuleTestCase {
         assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(GermanAnalyzer.class)));
         // analyzer types are bound early before we resolve aliases
         assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(StandardAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar_search.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasDefault() throws IOException {
@@ -168,6 +176,8 @@ public class AnalysisModuleTests extends ModuleTestCase {
         IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);
         assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
         assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasMoreThanOnce() throws IOException {
@@ -183,6 +193,10 @@ public class AnalysisModuleTests extends ModuleTestCase {
         AnalysisRegistry newRegistry = getNewRegistry(settings);
         IllegalStateException ise = expectThrows(IllegalStateException.class, () -> getIndexAnalyzers(newRegistry, settings));
         assertEquals("alias [default] is already used by [foobar]", ise.getMessage());
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar1.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasNotAllowedPost5x() throws IOException {
@@ -353,6 +367,8 @@ public class AnalysisModuleTests extends ModuleTestCase {
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("analyzer name must not start with '_'. got \"_invalid_name\""));
         }
+        assertWarnings("setting [index.analysis.analyzer.valid_name.alias] is only allowed on index [test] because it was " +
+                "created before 5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testDeprecatedPositionOffsetGap() throws IOException {

core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java

@@ -324,6 +324,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                 assertEquals(2, searchSourceBuilder.indexBoosts().size());
                 assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0));
                 assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1));
+                assertWarnings("Object format in indices_boost is deprecated, please use array format instead");
             }
         }
 

core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java

@@ -25,9 +25,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -67,22 +65,16 @@ import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.watcher.ResourceWatcherService;
-import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
 
 public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends ESTestCase {
 
@@ -91,7 +83,6 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     static IndicesQueriesRegistry indicesQueriesRegistry;
     private static ScriptService scriptService;
-    private ThreadContext threadContext;
 
     @BeforeClass
     public static void init() throws IOException {
@@ -123,39 +114,6 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         indicesQueriesRegistry = null;
     }
 
-    @Before
-    public void beforeTest() {
-        this.threadContext = new ThreadContext(Settings.EMPTY);
-        DeprecationLogger.setThreadContext(threadContext);
-    }
-
-    @After
-    public void afterTest() throws IOException {
-        //Check that there are no unaccounted warning headers. These should be checked with assertWarningHeaders(String...) in the
-        //appropriate test
-        final List<String> warnings = threadContext.getResponseHeaders().get(DeprecationLogger.DEPRECATION_HEADER);
-        assertNull("unexpected warning headers", warnings);
-        DeprecationLogger.removeThreadContext(this.threadContext);
-        this.threadContext.close();
-    }
-
-    protected void assertWarningHeaders(String... expectedWarnings) {
-        final List<String> actualWarnings = threadContext.getResponseHeaders().get(DeprecationLogger.DEPRECATION_HEADER);
-        assertThat(actualWarnings, hasSize(expectedWarnings.length));
-        for (String msg : expectedWarnings) {
-            assertThat(actualWarnings, hasItem(equalTo(msg)));
-        }
-        // "clear" current warning headers by setting a new ThreadContext
-        DeprecationLogger.removeThreadContext(this.threadContext);
-        try {
-            this.threadContext.close();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        this.threadContext = new ThreadContext(Settings.EMPTY);
-        DeprecationLogger.setThreadContext(this.threadContext);
-    }
-
     /** Returns random sort that is put under test */
     protected abstract T createTestItem();
 

core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java

@@ -275,7 +275,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
 
         QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.EMPTY);
         GeoDistanceSortBuilder.fromXContent(context, "");
-        assertWarningHeaders("Deprecated field [coerce] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     public void testIgnoreMalformedIsDeprecated() throws IOException {
@@ -294,7 +294,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
 
         QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.EMPTY);
         GeoDistanceSortBuilder.fromXContent(context, "");
-        assertWarningHeaders("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     public void testSortModeSumIsRejectedInJSON() throws IOException {
@@ -452,7 +452,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         sortBuilder.field("sort_mode", "max");
         sortBuilder.endObject();
         parse(sortBuilder);
-        assertWarningHeaders("Deprecated field [sort_mode] used, expected [mode] instead");
+        assertWarnings("Deprecated field [sort_mode] used, expected [mode] instead");
     }
 
     private GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws Exception {

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java

@@ -294,6 +294,7 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
         SearchResponse sr = client().prepareSearch().setQuery(builder)
                 .execute().actionGet();
         assertHitCount(sr, 1);
+        assertWarnings("[template] query is deprecated, use search template api instead");
     }
 
     // Relates to #10397
@@ -306,39 +307,36 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
                 .get();
         client().admin().indices().prepareRefresh().get();
 
-        int iterations = randomIntBetween(2, 11);
+        assertAcked(client().admin().cluster().preparePutStoredScript()
-        for (int i = 1; i < iterations; i++) {
+                .setScriptLang(MustacheScriptEngineService.NAME)
-            assertAcked(client().admin().cluster().preparePutStoredScript()
+                .setId("git01")
-                    .setScriptLang(MustacheScriptEngineService.NAME)
+                .setSource(new BytesArray("{\"template\":{\"query\": {\"match_phrase_prefix\": " +
-                    .setId("git01")
+                        "{\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
-                    .setSource(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
+                        "\"unsupported\": \"unsupported\"}}}}}")));
-                            "\"type\": \"ooophrase_prefix\"}}}}}")));
 
-            GetStoredScriptResponse getResponse = client().admin().cluster()
+        GetStoredScriptResponse getResponse = client().admin().cluster()
-                    .prepareGetStoredScript(MustacheScriptEngineService.NAME, "git01").get();
+                .prepareGetStoredScript(MustacheScriptEngineService.NAME, "git01").get();
-            assertNotNull(getResponse.getStoredScript());
+        assertNotNull(getResponse.getStoredScript());
 
-            Map<String, Object> templateParams = new HashMap<>();
+        Map<String, Object> templateParams = new HashMap<>();
-            templateParams.put("P_Keyword1", "dev");
+        templateParams.put("P_Keyword1", "dev");
 
-            ParsingException e = expectThrows(ParsingException.class, () -> new SearchTemplateRequestBuilder(client())
+        ParsingException e = expectThrows(ParsingException.class, () -> new SearchTemplateRequestBuilder(client())
-                    .setRequest(new SearchRequest("testindex").types("test"))
+                .setRequest(new SearchRequest("testindex").types("test"))
-                    .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
+                .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
-                    .get());
+                .get());
-            assertThat(e.getMessage(), containsString("[match] query does not support type ooophrase_prefix"));
+        assertThat(e.getMessage(), containsString("[match_phrase_prefix] query does not support [unsupported]"));
 
-            assertAcked(client().admin().cluster().preparePutStoredScript()
+        assertAcked(client().admin().cluster().preparePutStoredScript()
-                    .setScriptLang(MustacheScriptEngineService.NAME)
+                .setScriptLang(MustacheScriptEngineService.NAME)
-                    .setId("git01")
+                .setId("git01")
-                    .setSource(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
+                .setSource(new BytesArray("{\"query\": {\"match_phrase_prefix\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"}}}}")));
-                            "\"type\": \"phrase_prefix\"}}}}")));
 
-            SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
+        SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
-                    .setRequest(new SearchRequest("testindex").types("test"))
+                .setRequest(new SearchRequest("testindex").types("test"))
-                    .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
+                .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
-                    .get();
+                .get();
-            assertHitCount(searchResponse.getResponse(), 1);
+        assertHitCount(searchResponse.getResponse(), 1);
-        }
     }
 
     public void testIndexedTemplateWithArray() throws Exception {

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java

@@ -59,11 +59,12 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
     private QueryBuilder templateBase;
 
     /**
-     * All tests create deprecation warnings when an new {@link TemplateQueryBuilder} is created.
+     * All tests in this class cause deprecation warnings when a new {@link TemplateQueryBuilder} is created.
-     * Instead of having to check them once in every single test, this is done here after each test is run
+     * Instead of having to check them in every single test, we do it after each test is run
      */
-    @After void checkWarningHeaders() throws IOException {
+    @After
-        assertWarningHeaders("[template] query is deprecated, use search template api instead");
+    public void checkWarning() throws IOException {
+        assertWarnings("[template] query is deprecated, use search template api instead");
     }
 
     @Override

plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java

@@ -23,17 +23,23 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+
 public class FileBasedDiscoveryPluginTests extends ESTestCase {
 
-    public void testHostsProviderBwc() {
+    public void testHostsProviderBwc() throws IOException {
         FileBasedDiscoveryPlugin plugin = new FileBasedDiscoveryPlugin(Settings.EMPTY);
         Settings additionalSettings = plugin.additionalSettings();
         assertEquals("file", additionalSettings.get(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()));
+        assertWarnings("Using discovery.type setting to set hosts provider is deprecated. " +
+                "Set \"discovery.zen.hosts_provider: file\" instead");
     }
 
-    public void testHostsProviderExplicit() {
+    public void testHostsProviderExplicit() throws IOException {
         Settings settings = Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "foo").build();
         FileBasedDiscoveryPlugin plugin = new FileBasedDiscoveryPlugin(settings);
         assertEquals(Settings.EMPTY, plugin.additionalSettings());
+        assertWarnings("Using discovery.type setting to set hosts provider is deprecated. " +
+                "Set \"discovery.zen.hosts_provider: file\" instead");
     }
 }

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java

@@ -19,8 +19,6 @@
 
 package org.elasticsearch.repositories.s3;
 
-import java.io.IOException;
-
 import com.amazonaws.Protocol;
 import com.amazonaws.services.s3.AbstractAmazonS3;
 import com.amazonaws.services.s3.AmazonS3;
@@ -34,6 +32,8 @@ import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
+
 import static org.elasticsearch.repositories.s3.S3Repository.Repositories;
 import static org.elasticsearch.repositories.s3.S3Repository.Repository;
 import static org.elasticsearch.repositories.s3.S3Repository.getValue;
@@ -111,11 +111,14 @@ public class S3RepositoryTests extends ESTestCase {
             .put(Repository.BASE_PATH_SETTING.getKey(), "/foo/bar").build());
         S3Repository s3repo = new S3Repository(metadata, Settings.EMPTY, new DummyS3Service());
         assertEquals("foo/bar/", s3repo.basePath().buildAsString()); // make sure leading `/` is removed and trailing is added
-
+        assertWarnings("S3 repository base_path" +
+                " trimming the leading `/`, and leading `/` will not be supported for the S3 repository in future releases");
         metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.EMPTY);
         Settings settings = Settings.builder().put(Repositories.BASE_PATH_SETTING.getKey(), "/foo/bar").build();
         s3repo = new S3Repository(metadata, settings, new DummyS3Service());
         assertEquals("foo/bar/", s3repo.basePath().buildAsString()); // make sure leading `/` is removed and trailing is added
+        assertWarnings("S3 repository base_path" +
+                " trimming the leading `/`, and leading `/` will not be supported for the S3 repository in future releases");
     }
 
     public void testDefaultBufferSize() {

qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java

@@ -96,6 +96,7 @@ public class EvilLoggerTests extends ESTestCase {
             Level.WARN,
             "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
             "This is a deprecation message");
+        assertWarnings("This is a deprecation message");
     }
 
     public void testFindAppender() throws IOException, UserException {

test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java

@@ -47,13 +47,11 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -124,8 +122,6 @@ import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 
 public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> extends ESTestCase {
@@ -156,11 +152,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     private static String[] currentTypes;
     private static String[] randomTypes;
 
-    /**
-     * used to check warning headers of the deprecation logger
-     */
-    private ThreadContext threadContext;
-
     protected static Index getIndex() {
         return index;
     }
@@ -213,8 +204,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             serviceHolder = new ServiceHolder(nodeSettings, indexSettings, getPlugins(), this);
         }
         serviceHolder.clientInvocationHandler.delegate = this;
-        this.threadContext = new ThreadContext(Settings.EMPTY);
-        DeprecationLogger.setThreadContext(threadContext);
     }
 
     private static SearchContext getSearchContext(String[] types, QueryShardContext context) {
@@ -235,13 +224,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
     @After
     public void afterTest() throws IOException {
-        //Check that there are no unaccounted warning headers. These should be checked with {@link #checkWarningHeaders(String...)} in the
-        //appropriate test
-        final List<String> warnings = threadContext.getResponseHeaders().get(DeprecationLogger.DEPRECATION_HEADER);
-        assertNull("unexpected warning headers", warnings);
-        DeprecationLogger.removeThreadContext(this.threadContext);
-        this.threadContext.close();
-
         serviceHolder.clientInvocationHandler.delegate = null;
     }
 
@@ -1020,23 +1002,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         return query;
     }
 
-    protected void assertWarningHeaders(String... expectedWarnings) {
-        final List<String> actualWarnings = threadContext.getResponseHeaders().get(DeprecationLogger.DEPRECATION_HEADER);
-        assertThat(actualWarnings, hasSize(expectedWarnings.length));
-        for (String msg : expectedWarnings) {
-            assertThat(actualWarnings, hasItem(equalTo(msg)));
-        }
-        // "clear" current warning headers by setting a new ThreadContext
-        DeprecationLogger.removeThreadContext(this.threadContext);
-        try {
-            this.threadContext.close();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        this.threadContext = new ThreadContext(Settings.EMPTY);
-        DeprecationLogger.setThreadContext(this.threadContext);
-    }
-
     private static class ServiceHolder implements Closeable {
 
         private final IndicesQueriesRegistry indicesQueriesRegistry;

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -339,6 +339,13 @@ public abstract class ESIntegTestCase extends ESTestCase {
         initializeSuiteScope();
     }
 
+    @Override
+    protected final boolean enableWarningsCheck() {
+        //In an integ test it doesn't make sense to keep track of warnings: if the cluster is external the warnings are in another jvm,
+        //if the cluster is internal the deprecation logger is shared across all nodes
+        return false;
+    }
+
     protected final void beforeInternal() throws Exception {
         final Scope currentClusterScope = getCurrentClusterScope();
         switch (currentClusterScope) {

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -57,12 +57,14 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -129,6 +131,8 @@ import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
 
 /**
  * Base testcase for randomized unit testing with Elasticsearch
@@ -173,6 +177,7 @@ public abstract class ESTestCase extends LuceneTestCase {
     }
 
     protected final Logger logger = Loggers.getLogger(getClass());
+    private ThreadContext threadContext;
 
     // -----------------------------------------------------------------
     // Suite and test case setup/cleanup.
@@ -251,16 +256,62 @@ public abstract class ESTestCase extends LuceneTestCase {
     @Before
     public final void before()  {
         logger.info("[{}]: before test", getTestName());
+        if (enableWarningsCheck()) {
+            this.threadContext = new ThreadContext(Settings.EMPTY);
+            DeprecationLogger.setThreadContext(threadContext);
+        }
+    }
+
+    /**
+     * Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax
+     * was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}.
+     */
+    protected boolean enableWarningsCheck() {
+        return true;
     }
 
     @After
     public final void after() throws Exception {
         checkStaticState();
+        if (enableWarningsCheck()) {
+            ensureNoWarnings();
+        }
         ensureAllSearchContextsReleased();
         ensureCheckIndexPassed();
         logger.info("[{}]: after test", getTestName());
     }
 
+    private void ensureNoWarnings() throws IOException {
+        //Check that there are no unaccounted warning headers. These should be checked with {@link #checkWarningHeaders(String...)} in the
+        //appropriate test
+        try {
+            final List<String> warnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER);
+            assertNull("unexpected warning headers", warnings);
+        } finally {
+            DeprecationLogger.removeThreadContext(this.threadContext);
+            this.threadContext.close();
+        }
+    }
+
+    protected final void assertWarnings(String... expectedWarnings) throws IOException {
+        if (enableWarningsCheck() == false) {
+            throw new IllegalStateException("unable to check warning headers if the test is not set to do so");
+        }
+        try {
+            final List<String> actualWarnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER);
+            assertThat(actualWarnings, hasSize(expectedWarnings.length));
+            for (String msg : expectedWarnings) {
+                assertThat(actualWarnings, hasItem(equalTo(msg)));
+            }
+        } finally {
+            // "clear" current warning headers by setting a new ThreadContext
+            DeprecationLogger.removeThreadContext(this.threadContext);
+            this.threadContext.close();
+            this.threadContext = new ThreadContext(Settings.EMPTY);
+            DeprecationLogger.setThreadContext(this.threadContext);
+        }
+    }
+
     private static final List<StatusData> statusData = new ArrayList<>();
     static {
         // ensure that the status logger is set to the warn level so we do not miss any warnings with our Log4j usage