From a1e8fb794cc454b8b89f71aa4e593786192a2a29 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 15 Apr 2016 14:43:04 -0600 Subject: [PATCH] Allow JSON with unquoted field names by enabling system property In Elasticsearch 5.0.0, by default unquoted field names in JSON will be rejected. This can cause issues, however, for documents that were already indexed with unquoted field names. To alleviate this, a system property has been added that can be enabled so migration can occur. This system property will be removed in Elasticsearch 6.0.0 Resolves #17674 --- .../common/xcontent/json/JsonXContent.java | 21 ++++++++++++++++++- .../java/org/elasticsearch/node/Node.java | 8 +++++++ .../src/main/resources/config/jvm.options | 8 +++++++ .../migration/migrate_5_0/rest.asciidoc | 11 ++++++++-- .../packaging/scripts/20_tar_package.bats | 18 ++++++++++++++++ 5 files changed, 63 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index 54fe6bab458..f991a462f3b 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -46,11 +46,30 @@ public class JsonXContent implements XContent { } private final static JsonFactory jsonFactory; + public final static String JSON_ALLOW_UNQUOTED_FIELD_NAMES = "elasticsearch.json.allow_unquoted_field_names"; public final static JsonXContent jsonXContent; + public final static boolean unquotedFieldNamesSet; static { jsonFactory = new JsonFactory(); - jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false); + // TODO: Remove the system property configuration for this in Elasticsearch 6.0.0 + String jsonUnquoteProp = System.getProperty(JSON_ALLOW_UNQUOTED_FIELD_NAMES); + if (jsonUnquoteProp == null) { + unquotedFieldNamesSet = false; + jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false); + } else { + unquotedFieldNamesSet = true; + switch (jsonUnquoteProp) { + case "true": + jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + break; + case "false": + jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false); + break; + default: + throw new IllegalArgumentException("invalid value for [" + JSON_ALLOW_UNQUOTED_FIELD_NAMES + "]: " + jsonUnquoteProp); + } + } jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true); jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true); jsonFactory.configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now... diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 4590081fcea..d543d116c8f 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -47,6 +47,7 @@ import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.network.NetworkAddress; @@ -60,6 +61,7 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoverySettings; @@ -174,6 +176,12 @@ public class Node implements Closeable { logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]", tmpEnv.configFile(), Arrays.toString(tmpEnv.dataFiles()), tmpEnv.logsFile(), tmpEnv.pluginsFile()); } + // TODO: Remove this in Elasticsearch 6.0.0 + if (JsonXContent.unquotedFieldNamesSet) { + DeprecationLogger dLogger = new DeprecationLogger(logger); + dLogger.deprecated("[{}] has been set, but will be removed in Elasticsearch 6.0.0", + JsonXContent.JSON_ALLOW_UNQUOTED_FIELD_NAMES); + } this.pluginsService = new PluginsService(tmpSettings, tmpEnv.modulesFile(), tmpEnv.pluginsFile(), classpathPlugins); this.settings = pluginsService.updatedSettings(); diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options index e82b9c245c2..726f2772db6 100644 --- a/distribution/src/main/resources/config/jvm.options +++ b/distribution/src/main/resources/config/jvm.options @@ -79,3 +79,11 @@ # log GC status to a file with time stamps # ensure the directory exists #-Xloggc:${loggc} + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true diff --git a/docs/reference/migration/migrate_5_0/rest.asciidoc b/docs/reference/migration/migrate_5_0/rest.asciidoc index b95b0cc43e7..439fab4d994 100644 --- a/docs/reference/migration/migrate_5_0/rest.asciidoc +++ b/docs/reference/migration/migrate_5_0/rest.asciidoc @@ -38,5 +38,12 @@ empty, that means that the node is a coordinating only node. ==== Forbid unquoted JSON -Previously, JSON documents were allowed with unquoted field names, which -isn't strictly JSON and broke some Elasticsearch clients. +Previously, JSON documents were allowed with unquoted field names, which isn't +strictly JSON and broke some Elasticsearch clients. If documents were already +indexed with unquoted fields in a previous vesrion of Elasticsearch, some +operations may throw errors. To accompany this, a commented out JVM option has +been added to the `jvm.options` file: +`-Delasticsearch.json.allow_unquoted_field_names`. + +Note that this option is provided solely for migration purposes and will be +removed in Elasticsearch 6.0.0. diff --git a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats index 5be4d2c6c64..f6f4d3e76d0 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats @@ -115,6 +115,24 @@ setup() { export ES_JAVA_OPTS=$es_java_opts } +@test "[TAR]" start Elasticsearch with unquoted JSON option { + local es_java_opts=$ES_JAVA_OPTS + local es_jvm_options=$ES_JVM_OPTIONS + local temp=`mktemp -d` + touch "$temp/jvm.options" + chown -R elasticsearch:elasticsearch "$temp" + echo "-Delasticsearch.json.allow_unquoted_field_names=true" >> "$temp/jvm.options" + export ES_JVM_OPTIONS="$temp/jvm.options" + start_elasticsearch_service + # unquoted field name + curl -s -XPOST localhost:9200/i/d/1 -d'{foo: "bar"}' + [ "$?" -eq 0 ] + curl -s -XDELETE localhost:9200/i + stop_elasticsearch_service + export ES_JVM_OPTIONS=$es_jvm_options + export ES_JAVA_OPTS=$es_java_opts +} + @test "[TAR]" remove tar { rm -rf "/tmp/elasticsearch" }