[ML] Remove "8" prefixes from file structure finder timestamp formats (#38016)

In 7.x Java timestamp formats are the default timestamp format and
there is no need to prefix them with "8".  (The "8" prefix was used
in 6.7 to distinguish Java timestamp formats from Joda timestamp
formats.)

This change removes the "8" prefixes from timestamp formats in the
output of the ML file structure finder.

docs/reference/ml/apis/find-file-structure.asciidoc

@@ -606,11 +606,11 @@ If the request does not encounter errors, you receive the following result:
     },
     "tpep_dropoff_datetime" : {
       "type" : "date",
-      "format" : "8yyyy-MM-dd HH:mm:ss"
+      "format" : "yyyy-MM-dd HH:mm:ss"
     },
     "tpep_pickup_datetime" : {
       "type" : "date",
-      "format" : "8yyyy-MM-dd HH:mm:ss"
+      "format" : "yyyy-MM-dd HH:mm:ss"
     },
     "trip_distance" : {
       "type" : "double"
@@ -624,7 +624,7 @@ If the request does not encounter errors, you receive the following result:
           "field" : "tpep_pickup_datetime",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd HH:mm:ss"
+            "yyyy-MM-dd HH:mm:ss"
           ]
         }
       }
@@ -1398,7 +1398,7 @@ this:
           "field" : "timestamp",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd'T'HH:mm:ss,SSS"
+            "yyyy-MM-dd'T'HH:mm:ss,SSS"
           ]
         }
       },
@@ -1558,7 +1558,7 @@ this:
           "field" : "timestamp",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd'T'HH:mm:ss,SSS"
+            "yyyy-MM-dd'T'HH:mm:ss,SSS"
           ]
         }
       },

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java

@@ -353,7 +353,7 @@ public final class FileStructureUtils {
             if (needClientTimezone) {
                 dateProcessorSettings.put("timezone", "{{ " + BEAT_TIMEZONE_FIELD + " }}");
             }
-            dateProcessorSettings.put("formats", jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats));
+            dateProcessorSettings.put("formats", timestampFormats);
             processors.add(Collections.singletonMap("date", dateProcessorSettings));
         }
 
@@ -365,19 +365,4 @@ public final class FileStructureUtils {
         pipeline.put(Pipeline.PROCESSORS_KEY, processors);
         return pipeline;
     }
-
-    // TODO: remove this method when Java time formats are the default
-    static List<String> jodaBwcJavaTimestampFormatsForIngestPipeline(List<String> javaTimestampFormats) {
-        return javaTimestampFormats.stream().map(format -> {
-            switch (format) {
-                case "ISO8601":
-                case "UNIX_MS":
-                case "UNIX":
-                case "TAI64N":
-                    return format;
-                default:
-                    return "8" + format;
-            }
-        }).collect(Collectors.toList());
-    }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java

@@ -472,8 +472,7 @@ public final class TimestampFormatFinder {
                     case "UNIX":
                         return Stream.of("epoch_second");
                     default:
-                        // TODO: remove the "8" prefix when Java time formats are the default
+                        return Stream.of(format);
-                        return Stream.of("8" + format);
                 }
             }).collect(Collectors.joining("||"));
             if (formats.isEmpty() == false) {

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java

@@ -331,8 +331,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("foo"));
         Map<String, String> expectedTimeMapping = new HashMap<>();
         expectedTimeMapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, "date");
-        // TODO: remove the "8" prefix when Java time formats are the default
+        expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "yyyy-MM-dd HH:mm:ss,SSS");
-        expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "8" + "yyyy-MM-dd HH:mm:ss,SSS");
         assertEquals(expectedTimeMapping, mappings.get("time"));
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("bar"));
         assertNull(mappings.get("nothing"));
@@ -372,8 +371,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
         assertNotNull(dateProcessor);
         assertEquals(timestampField, dateProcessor.get("field"));
         assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
-        // TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
+        assertEquals(timestampFormats, dateProcessor.get("formats"));
-        assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
 
         // After removing the two expected fields there should be nothing left in the pipeline
         assertEquals(Collections.emptyMap(), pipeline);
@@ -406,8 +404,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
         assertNotNull(dateProcessor);
         assertEquals(timestampField, dateProcessor.get("field"));
         assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
-        // TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
+        assertEquals(timestampFormats, dateProcessor.get("formats"));
-        assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
 
         Map<String, Object> removeProcessor = (Map<String, Object>) processors.get(2).get("remove");
         assertNotNull(removeProcessor);