Upgrade jackson to 2.10.3 and GeoIP to 2.13.1 (#53642)

Re-applies the change from #53523 along with test fixes.

closes #53626
closes #53624
closes #53622
closes #53625

Co-authored-by: Nik Everett <nik9000@gmail.com>
Co-authored-by: Lee Hinman <dakrone@users.noreply.github.com>
Co-authored-by: Jake Landis <jake.landis@elastic.co>

buildSrc/version.properties

@@ -7,12 +7,8 @@ bundled_jdk = 13.0.2+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-# note that ingest-geoip has a hard-coded version; if you modify this version,
+jackson           = 2.10.3
-# you should also inspect that version to see if it can be advanced along with
+snakeyaml         = 1.24
-# the com.maxmind.geoip2:geoip2 dependency
-jackson           = 2.8.11
-jacksondatabind   = 2.8.11.6
-snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc

client/sniffer/licenses/jackson-core-2.10.3.jar.sha1

@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2

client/sniffer/licenses/jackson-core-2.8.11.jar.sha1

@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b

libs/x-content/build.gradle

@@ -49,6 +49,7 @@ forbiddenApisMain {
 thirdPartyAudit.ignoreMissingClasses(
   // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
   'com.fasterxml.jackson.databind.ObjectMapper',
+  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
 )
 
 dependencyLicenses {

libs/x-content/licenses/jackson-core-2.10.3.jar.sha1

@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2

libs/x-content/licenses/jackson-core-2.8.11.jar.sha1

@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b

libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1

@@ -0,0 +1 @@
+1ba01fef9c3b7ed388d91e71dc733b315c7374cd

libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1

@@ -1 +0,0 @@
-8b9826e16c3366764bfb7ad7362554f0471046c3

libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1

@@ -0,0 +1 @@
+ff397547ff168e77279a1cd549e2ca4923c991aa

libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1

@@ -1 +0,0 @@
-d9d1c49c5d9d5e46e2aee55f3cdd119286fe0fc1

libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1

@@ -0,0 +1 @@
+4dc1a172812d9da27c1afd6a08f4f12aad7b14dd

libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1

@@ -1 +0,0 @@
-2e77c6ff7342cd61ab1ae7cb14ed16aebfc8a72a

libs/x-content/licenses/snakeyaml-1.17.jar.sha1

@@ -1 +0,0 @@
-7a27ea250c5130b2922b86dea63cbb1cc10a660c

libs/x-content/licenses/snakeyaml-1.24.jar.sha1

@@ -0,0 +1 @@
+13a9c0d6776483c3876e3ff9384f9bb55b17001b

libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.support.MapXContentParser;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.EnumSet;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentParserTests.generateRandomObject;
@@ -73,7 +74,13 @@ public class MapXContentParserTests extends ESTestCase {
     }
 
     public void compareTokens(CheckedConsumer<XContentBuilder, IOException> consumer) throws IOException {
-        final XContentType xContentType = randomFrom(XContentType.values());
+        for (XContentType xContentType : EnumSet.allOf(XContentType.class)) {
+            logger.info("--> testing with xcontent type: {}", xContentType);
+            compareTokens(consumer, xContentType);
+        }
+    }
+
+    public void compareTokens(CheckedConsumer<XContentBuilder, IOException> consumer, XContentType xContentType) throws IOException {
         try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
             consumer.accept(builder);
             final Map<String, Object> map;
@@ -94,7 +101,13 @@ public class MapXContentParserTests extends ESTestCase {
                         assertEquals(token, mapToken);
                         assertEquals(parser.currentName(), mapParser.currentName());
                         if (token != null && (token.isValue() || token == XContentParser.Token.VALUE_NULL)) {
-                            assertEquals(parser.textOrNull(), mapParser.textOrNull());
+                            if (xContentType != XContentType.YAML || token != XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+                                // YAML struggles with converting byte arrays into text, because it
+                                // does weird base64 decoding to the values. We don't do this
+                                // weirdness in the MapXContentParser, so don't try to stringify it.
+                                // The .binaryValue() comparison below still works correctly though.
+                                assertEquals(parser.textOrNull(), mapParser.textOrNull());
+                            }
                             switch (token) {
                                 case VALUE_STRING:
                                     assertEquals(parser.text(), mapParser.text());

libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java

@@ -78,14 +78,17 @@ public class XContentParserTests extends ESTestCase {
 
             assertEquals(value, number.floatValue(), 0.0f);
 
-            if (xContentType == XContentType.CBOR) {
+            switch (xContentType) {
-                // CBOR parses back a float
+                case CBOR:
-                assertTrue(number instanceof Float);
+                case SMILE:
-            } else {
+                    assertThat(number, instanceOf(Float.class));
-                // JSON, YAML and SMILE parses back the float value as a double
+                    break;
-                // This will change for SMILE in Jackson 2.9 where all binary based
+                case JSON:
-                // formats will return a float
+                case YAML:
-                assertTrue(number instanceof Double);
+                    assertThat(number, instanceOf(Double.class));
+                    break;
+                default:
+                    throw new AssertionError("unexpected x-content type [" + xContentType + "]");
             }
         }
     }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java

@@ -65,7 +65,7 @@ public class JsonProcessorTests extends ESTestCase {
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
         assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " +
-            "was expecting ('true', 'false' or 'null')"));
+            "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"));
     }
 
     public void testByteArray() {
@@ -75,7 +75,12 @@ public class JsonProcessorTests extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
-        assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')"));
+        assertThat(
+            exception.getCause().getMessage(),
+            containsString(
+                "Unrecognized token 'B': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"
+            )
+        );
     }
 
     public void testNull() throws Exception {

modules/ingest-geoip/build.gradle

@@ -25,13 +25,11 @@ esplugin {
 }
 
 dependencies {
-  // Upgrade to 2.10.0 or higher when jackson-core gets upgraded to 2.9.x. Blocked by #27032
+  compile('com.maxmind.geoip2:geoip2:2.13.1')
-  compile('com.maxmind.geoip2:geoip2:2.9.0')
   // geoip2 dependencies:
-  // do not hardcode this to the version in version.properties, it needs to be upgraded separately with geoip2
+  compile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  compile("com.fasterxml.jackson.core:jackson-annotations:2.8.11")
+  compile("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
-  compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.6")
+  compile('com.maxmind.db:maxmind-db:1.3.1')
-  compile('com.maxmind.db:maxmind-db:1.2.2')
 
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
 }

modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1

@@ -0,0 +1 @@
+f27d1a49d5a29dd4a7ac5006ce2eb16b8b9bb888

modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1

@@ -1 +0,0 @@
-c12b463a2c10824225c0b27952c49b464cb0e4c6

modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1

@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7

modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1

@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f

modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1

@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439

modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1

@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368

modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1

@@ -1 +0,0 @@
-78c22a03de1e222b0751855aff7bb6e6db5569e5

modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1

@@ -0,0 +1 @@
+211bca628225bc0f719051b16deb03a747d7a14f

plugins/discovery-ec2/build.gradle

@@ -36,7 +36,7 @@ dependencies {
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 

plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1

@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7

plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1

@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f

plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1

@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439

plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1

@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368

plugins/repository-s3/build.gradle

@@ -43,7 +43,7 @@ dependencies {
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   compile "joda-time:joda-time:${versions.joda}"

plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1

@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7

plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1

@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f

plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1

@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439

plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1

@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368

server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java

@@ -91,11 +91,11 @@ public class XContentParserUtilsTests extends ESTestCase {
     public void testStoredFieldsValueBinary() throws IOException {
         final byte[] value = randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8");
         assertParseFieldsSimpleValue(value, (xcontentType, result) -> {
-            if (xcontentType == XContentType.JSON || xcontentType == XContentType.YAML) {
+            if (xcontentType == XContentType.JSON) {
-                //binary values will be parsed back and returned as base64 strings when reading from json and yaml
+                // binary values will be parsed back and returned as base64 strings when reading from json
                 assertArrayEquals(value, Base64.getDecoder().decode((String) result));
             } else {
-                //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
+                // cbor, smile, and yaml support binary
                 assertArrayEquals(value, ((BytesArray) result).array());
             }
         });

server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java

@@ -105,13 +105,13 @@ public class IndexingSlowLogTests extends ESTestCase {
         final UncheckedIOException e = expectThrows(UncheckedIOException.class,
             ()->new IndexingSlowLogMessage(index, doc, 10, true, 3));
         assertThat(e, hasToString(containsString("_failed_to_convert_[Unrecognized token 'invalid':"
-            + " was expecting ('true', 'false' or 'null')\\n"
+            + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\\n"
-            + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
+            + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
         assertNotNull(e.getCause());
         assertThat(e.getCause(), instanceOf(JsonParseException.class));
         assertThat(e.getCause(), hasToString(containsString("Unrecognized token 'invalid':"
-                + " was expecting ('true', 'false' or 'null')\n"
+                + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n"
-                + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
+                + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
     }
 
     public void testReformatSetting() {

test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java

@@ -136,20 +136,16 @@ public final class RandomObjects {
      */
     public static Object getExpectedParsedValue(XContentType xContentType, Object value) {
         if (value instanceof BytesArray) {
-            if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
+            if (xContentType == XContentType.JSON) {
-                //JSON and YAML write the base64 format
+                //JSON writes base64 format
                 return Base64.getEncoder().encodeToString(((BytesArray)value).toBytesRef().bytes);
             }
         }
         if (value instanceof Float) {
-            if (xContentType == XContentType.CBOR) {
+            if (xContentType == XContentType.CBOR || xContentType == XContentType.SMILE) {
-                //with CBOR we get back a float
+                // with binary content types we pass back the object as is
                 return value;
             }
-            if (xContentType == XContentType.SMILE) {
-                //with SMILE we get back a double (this will change in Jackson 2.9 where it will return a Float)
-                return ((Float)value).doubleValue();
-            }
             //with JSON AND YAML we get back a double, but with float precision.
             return Double.parseDouble(value.toString());
         }

x-pack/plugin/sql/jdbc/build.gradle

@@ -44,3 +44,8 @@ dependencyLicenses {
 shadowJar {
   relocate 'com.fasterxml', 'org.elasticsearch.fasterxml'
 }
+
+thirdPartyAudit.ignoreMissingClasses(
+  'com.fasterxml.jackson.databind.ObjectMapper',
+  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
+)

x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1

@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2

x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1

@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b

x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1

@@ -0,0 +1 @@
+1ba01fef9c3b7ed388d91e71dc733b315c7374cd

x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1

@@ -1 +0,0 @@
-8b9826e16c3366764bfb7ad7362554f0471046c3

x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1

@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2

x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1

@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b

x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1

@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2

x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1

@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b

x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java

@@ -92,7 +92,8 @@ public class RemoteFailureTests extends ESTestCase {
     public void testInvalidJson() {
         IOException e = expectThrows(IOException.class, () -> parse("invalid_json.txt"));
         assertEquals(
-            "Can't parse error from Elasticsearch [Unrecognized token 'I': was expecting 'null', 'true', 'false' or NaN] "
+            "Can't parse error from Elasticsearch [Unrecognized token 'I': "
+                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] "
                 + "at [line 1 col 1]. Response:\n"
                 + "I'm not json at all",
             e.getMessage());
@@ -118,7 +119,7 @@ public class RemoteFailureTests extends ESTestCase {
             }).streamInput()));
         assertThat(e.getMessage(),
             startsWith("Can't parse error from Elasticsearch [Unrecognized token 'ÿ': "
-                + "was expecting ('true', 'false' or 'null')] at [line 1 col 1]. Response:\n"));
+                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] at [line 1 col 4]. Response:\n"));
     }
 
     public void testTooBig() {

x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1

@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2

x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1

@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b