honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Remove "8" prefixes from file structure finder timestamp formats (#38016) 

In 7.x Java timestamp formats are the default timestamp format and
there is no need to prefix them with "8".  (The "8" prefix was used
in 6.7 to distinguish Java timestamp formats from Joda timestamp
formats.)

This change removes the "8" prefixes from timestamp formats in the
output of the ML file structure finder.
This commit is contained in:
David Roberts 2019-02-01 15:36:04 +00:00 committed by GitHub
parent 5c58c2508e
commit 1fa413a16d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 10 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/reference/ml/apis/find-file-structure.asciidoc
View File

@ -606,11 +606,11 @@ If the request does not encounter errors, you receive the following result:
},
"tpep_dropoff_datetime" : {
"type" : "date",
"format" : "8yyyy-MM-dd HH:mm:ss"
"format" : "yyyy-MM-dd HH:mm:ss"
},
"tpep_pickup_datetime" : {
"type" : "date",
"format" : "8yyyy-MM-dd HH:mm:ss"
"format" : "yyyy-MM-dd HH:mm:ss"
},
"trip_distance" : {
"type" : "double"
@ -624,7 +624,7 @@ If the request does not encounter errors, you receive the following result:
"field" : "tpep_pickup_datetime",
"timezone" : "{{ beat.timezone }}",
"formats" : [
"8yyyy-MM-dd HH:mm:ss"
"yyyy-MM-dd HH:mm:ss"
]
}
}
@ -1398,7 +1398,7 @@ this:
"field" : "timestamp",
"timezone" : "{{ beat.timezone }}",
"formats" : [
"8yyyy-MM-dd'T'HH:mm:ss,SSS"
"yyyy-MM-dd'T'HH:mm:ss,SSS"
]
}
},
@ -1558,7 +1558,7 @@ this:
"field" : "timestamp",
"timezone" : "{{ beat.timezone }}",
"formats" : [
"8yyyy-MM-dd'T'HH:mm:ss,SSS"
"yyyy-MM-dd'T'HH:mm:ss,SSS"
]
}
},

17
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java
View File

@ -353,7 +353,7 @@ public final class FileStructureUtils {
if (needClientTimezone) {
dateProcessorSettings.put("timezone", "{{ " + BEAT_TIMEZONE_FIELD + " }}");
}
dateProcessorSettings.put("formats", jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats));
dateProcessorSettings.put("formats", timestampFormats);
processors.add(Collections.singletonMap("date", dateProcessorSettings));
}
@ -365,19 +365,4 @@ public final class FileStructureUtils {
pipeline.put(Pipeline.PROCESSORS_KEY, processors);
return pipeline;
}
// TODO: remove this method when Java time formats are the default
static List<String> jodaBwcJavaTimestampFormatsForIngestPipeline(List<String> javaTimestampFormats) {
return javaTimestampFormats.stream().map(format -> {
switch (format) {
case "ISO8601":
case "UNIX_MS":
case "UNIX":
case "TAI64N":
return format;
default:
return "8" + format;
}
}).collect(Collectors.toList());
}
}

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java
View File

@ -472,8 +472,7 @@ public final class TimestampFormatFinder {
case "UNIX":
return Stream.of("epoch_second");
default:
// TODO: remove the "8" prefix when Java time formats are the default
return Stream.of("8" + format);
return Stream.of(format);
}
}).collect(Collectors.joining("||"));
if (formats.isEmpty() == false) {

9
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java
View File

@ -331,8 +331,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("foo"));
Map<String, String> expectedTimeMapping = new HashMap<>();
expectedTimeMapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, "date");
// TODO: remove the "8" prefix when Java time formats are the default
expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "8" + "yyyy-MM-dd HH:mm:ss,SSS");
expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "yyyy-MM-dd HH:mm:ss,SSS");
assertEquals(expectedTimeMapping, mappings.get("time"));
assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("bar"));
assertNull(mappings.get("nothing"));
@ -372,8 +371,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
assertNotNull(dateProcessor);
assertEquals(timestampField, dateProcessor.get("field"));
assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
// TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
assertEquals(timestampFormats, dateProcessor.get("formats"));
// After removing the two expected fields there should be nothing left in the pipeline
assertEquals(Collections.emptyMap(), pipeline);
@ -406,8 +404,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
assertNotNull(dateProcessor);
assertEquals(timestampField, dateProcessor.get("field"));
assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
// TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
assertEquals(timestampFormats, dateProcessor.get("formats"));
Map<String, Object> removeProcessor = (Map<String, Object>) processors.get(2).get("remove");
assertNotNull(removeProcessor);