Add field with dot in name to 2.4+ static bwc indexes (#20360)

This change adds a `field.with.dots` to all 2.4 bwc indicse and above.
It also adds verification code to OldIndexBackwardsCompatibilityIT to
ensure we upgrade the indices cleanly and the field is present.

Closes #19956

core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java

@@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.indices.segments.IndexSegments;
 import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
 import org.elasticsearch.action.admin.indices.segments.ShardSegments;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -310,6 +311,14 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
         searchRsp = searchReq.get();
         ElasticsearchAssertions.assertNoFailures(searchRsp);
         assertEquals(numDocs, searchRsp.getHits().getTotalHits());
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(indexName).get();
+        Version versionCreated = Version.fromId(Integer.parseInt(getSettingsResponse.getSetting(indexName, "index.version.created")));
+        if (versionCreated.onOrAfter(Version.V_2_4_0)) {
+            searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.existsQuery("field.with.dots"));
+            searchRsp = searchReq.get();
+            ElasticsearchAssertions.assertNoFailures(searchRsp);
+            assertEquals(numDocs, searchRsp.getHits().getTotalHits());
+        }
     }
 
     boolean findPayloadBoostInExplanation(Explanation expl) {

dev-tools/create_bwc_index.py

@@ -58,28 +58,32 @@ def assert_sort(hits):
 
 # Indexes the given number of document into the given index
 # and randomly runs refresh, optimize and flush commands
-def index_documents(es, index_name, type, num_docs):
+def index_documents(es, index_name, type, num_docs, supports_dots_in_field_names):
   logging.info('Indexing %s docs' % num_docs)
-  for id in range(0, num_docs):
-    es.index(index=index_name, doc_type=type, id=id, body={'string': str(random.randint(0, 100)),
-                                                           'long_sort': random.randint(0, 100),
-                                                           'double_sort' : float(random.randint(0, 100)),
-                                                           'bool' : random.choice([True, False])})
-    if rarely():
-      es.indices.refresh(index=index_name)
-    if rarely():
-      es.indices.flush(index=index_name, force=frequently())
+  index(es, index_name, type, num_docs, supports_dots_in_field_names, True)
   logging.info('Flushing index')
   es.indices.flush(index=index_name)
 
-def reindex_docs(es, index_name, type, num_docs):
-  logging.info('Re-indexing %s docs' % num_docs)
-  # reindex some docs after the flush such that we have something in the translog
+def index(es, index_name, type, num_docs, supports_dots_in_field_names, flush=False):
   for id in range(0, num_docs):
-    es.index(index=index_name, doc_type=type, id=id, body={'string': str(random.randint(0, 100)),
+    body = {'string': str(random.randint(0, 100)),
             'long_sort': random.randint(0, 100),
             'double_sort' : float(random.randint(0, 100)),
-                                                           'bool' : random.choice([True, False])})
+            'bool' : random.choice([True, False])}
+    if supports_dots_in_field_names:
+      body['field.with.dots'] = str(random.randint(0, 100))
+
+    es.index(index=index_name, doc_type=type, id=id, body=body)
+
+    if rarely():
+      es.indices.refresh(index=index_name)
+    if rarely() and flush:
+      es.indices.flush(index=index_name, force=frequently())
+
+def reindex_docs(es, index_name, type, num_docs, supports_dots_in_field_names):
+  logging.info('Re-indexing %s docs' % num_docs)
+  # reindex some docs after the flush such that we have something in the translog
+  index(es, index_name, type, num_docs, supports_dots_in_field_names)
 
 def delete_by_query(es, version, index_name, doc_type):
 
@@ -158,7 +162,8 @@ def start_node(version, release_dir, data_dir, repo_dir, tcp_port=DEFAULT_TRANSP
   ]
   if version.startswith('0.') or version.startswith('1.0.0.Beta') :
     cmd.append('-f') # version before 1.0 start in background automatically
-  return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                          env=dict(os.environ, ES_JAVA_OPTS='-Dmapper.allow_dots_in_name=true'))
 
 def install_plugin(version, release_dir, plugin_name):
   run_plugin(version, release_dir, 'install', [plugin_name])
@@ -257,6 +262,16 @@ def generate_index(client, version, index_name):
         'auto_boost': True
       }
     }
+  mappings['doc'] = {'properties' : {}}
+  supports_dots_in_field_names = parse_version(version) >= parse_version("2.4.0")
+  if supports_dots_in_field_names:
+    mappings["doc"]['properties'].update({
+        'field.with.dots': {
+          'type': 'string',
+          'boost': 4
+        }
+      })
+
   if parse_version(version) < parse_version("5.0.0-alpha1"):
     mappings['norms'] = {
       'properties': {
@@ -300,14 +315,12 @@ def generate_index(client, version, index_name):
         }
       }
     }
-    mappings['doc'] = {
-      'properties': {
+    mappings['doc']['properties'].update({
         'string': {
           'type': 'text',
           'boost': 4
         }
-      }
-    }
+    })
 
   settings = {
     'number_of_shards': 1,
@@ -335,10 +348,10 @@ def generate_index(client, version, index_name):
     # lighter index for it to keep bw tests reasonable
     # see https://github.com/elastic/elasticsearch/issues/5817
     num_docs = int(num_docs / 10)
-  index_documents(client, index_name, 'doc', num_docs)
+  index_documents(client, index_name, 'doc', num_docs, supports_dots_in_field_names)
   logging.info('Running basic asserts on the data added')
   run_basic_asserts(client, index_name, 'doc', num_docs)
-  return num_docs
+  return num_docs, supports_dots_in_field_names
 
 def snapshot_index(client, version, repo_dir):
   persistent = {
@@ -448,7 +461,7 @@ def create_bwc_index(cfg, version):
     node = start_node(version, release_dir, data_dir, repo_dir, cfg.tcp_port, cfg.http_port)
     client = create_client(cfg.http_port)
     index_name = 'index-%s' % version.lower()
-    num_docs = generate_index(client, version, index_name)
+    num_docs, supports_dots_in_field_names = generate_index(client, version, index_name)
     if snapshot_supported:
       snapshot_index(client, version, repo_dir)
 
@@ -457,7 +470,7 @@ def create_bwc_index(cfg, version):
     # will already have the deletions applied on upgrade.
     if version.startswith('0.') or version.startswith('1.'):
       delete_by_query(client, version, index_name, 'doc')
-    reindex_docs(client, index_name, 'doc', min(100, num_docs))
+    reindex_docs(client, index_name, 'doc', min(100, num_docs), supports_dots_in_field_names)
 
     shutdown_node(node)
     node = None