Allow JSON with unquoted field names by enabling system property

In Elasticsearch 5.0.0, by default unquoted field names in JSON will be
rejected. This can cause issues, however, for documents that were
already indexed with unquoted field names. To alleviate this, a system
property has been added that can be enabled so migration can occur.

This system property will be removed in Elasticsearch 6.0.0

Resolves #17674

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java

@@ -46,11 +46,30 @@ public class JsonXContent implements XContent {
     }
 
     private final static JsonFactory jsonFactory;
+    public final static String JSON_ALLOW_UNQUOTED_FIELD_NAMES = "elasticsearch.json.allow_unquoted_field_names";
     public final static JsonXContent jsonXContent;
+    public final static boolean unquotedFieldNamesSet;
 
     static {
         jsonFactory = new JsonFactory();
-        jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false);
+        // TODO: Remove the system property configuration for this in Elasticsearch 6.0.0
+        String jsonUnquoteProp = System.getProperty(JSON_ALLOW_UNQUOTED_FIELD_NAMES);
+        if (jsonUnquoteProp == null) {
+            unquotedFieldNamesSet = false;
+            jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false);
+        } else {
+            unquotedFieldNamesSet = true;
+            switch (jsonUnquoteProp) {
+                case "true":
+                    jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+                    break;
+                case "false":
+                    jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false);
+                    break;
+                default:
+                    throw new IllegalArgumentException("invalid value for [" + JSON_ALLOW_UNQUOTED_FIELD_NAMES + "]: " + jsonUnquoteProp);
+            }
+        }
         jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
         jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
         jsonFactory.configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...

core/src/main/java/org/elasticsearch/node/Node.java

@@ -47,6 +47,7 @@ import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -60,6 +61,7 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
@@ -174,6 +176,12 @@ public class Node implements Closeable {
             logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]",
                     tmpEnv.configFile(), Arrays.toString(tmpEnv.dataFiles()), tmpEnv.logsFile(), tmpEnv.pluginsFile());
         }
+        // TODO: Remove this in Elasticsearch 6.0.0
+        if (JsonXContent.unquotedFieldNamesSet) {
+            DeprecationLogger dLogger = new DeprecationLogger(logger);
+            dLogger.deprecated("[{}] has been set, but will be removed in Elasticsearch 6.0.0",
+                    JsonXContent.JSON_ALLOW_UNQUOTED_FIELD_NAMES);
+        }
 
         this.pluginsService = new PluginsService(tmpSettings, tmpEnv.modulesFile(), tmpEnv.pluginsFile(), classpathPlugins);
         this.settings = pluginsService.updatedSettings();

distribution/src/main/resources/config/jvm.options

@@ -79,3 +79,11 @@
 # log GC status to a file with time stamps
 # ensure the directory exists
 #-Xloggc:${loggc}
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true

docs/reference/migration/migrate_5_0/rest.asciidoc

@@ -38,5 +38,12 @@ empty, that means that the node is a coordinating only node.
 
 ==== Forbid unquoted JSON
 
-Previously, JSON documents were allowed with unquoted field names, which
-isn't strictly JSON and broke some Elasticsearch clients.
+Previously, JSON documents were allowed with unquoted field names, which isn't
+strictly JSON and broke some Elasticsearch clients. If documents were already
+indexed with unquoted fields in a previous vesrion of Elasticsearch, some
+operations may throw errors. To accompany this, a commented out JVM option has
+been added to the `jvm.options` file:
+`-Delasticsearch.json.allow_unquoted_field_names`.
+
+Note that this option is provided solely for migration purposes and will be
+removed in Elasticsearch 6.0.0.

qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats

@@ -115,6 +115,24 @@ setup() {
     export ES_JAVA_OPTS=$es_java_opts
 }
 
+@test "[TAR]" start Elasticsearch with unquoted JSON option {
+    local es_java_opts=$ES_JAVA_OPTS
+    local es_jvm_options=$ES_JVM_OPTIONS
+    local temp=`mktemp -d`
+    touch "$temp/jvm.options"
+    chown -R elasticsearch:elasticsearch "$temp"
+    echo "-Delasticsearch.json.allow_unquoted_field_names=true" >> "$temp/jvm.options"
+    export ES_JVM_OPTIONS="$temp/jvm.options"
+    start_elasticsearch_service
+    # unquoted field name
+    curl -s -XPOST localhost:9200/i/d/1 -d'{foo: "bar"}'
+    [ "$?" -eq 0 ]
+    curl -s -XDELETE localhost:9200/i
+    stop_elasticsearch_service
+    export ES_JVM_OPTIONS=$es_jvm_options
+    export ES_JAVA_OPTS=$es_java_opts
+}
+
 @test "[TAR]" remove tar {
     rm -rf "/tmp/elasticsearch"
 }