honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Fix 2 digit year regex in find_file_structure (#51469) 

The DATE and DATESTAMP Grok patterns match 2 digit years
as well as 4 digit years.  The pattern determination in
find_file_structure worked correctly in this case, but
the regex used to create a multi-line start pattern was
assuming a 4 digit year.  Also, the quick rule-out
patterns did not always correctly consider 2 digit years,
meaning that detection was inconsistent.

This change fixes both problems, and also extends the
tests for DATE and DATESTAMP to check both 2 and 4 digit
years.
This commit is contained in:
David Roberts 2020-01-27 17:05:59 +00:00
parent 8559ff7cee
commit 3c223ceea1
2 changed files with 184 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java
View File

@ -159,13 +159,18 @@ public final class TimestampFormatFinder {
"%{MONTH} +%{MONTHDAY} %{YEAR} %{HOUR}:%{MINUTE}:(?:[0-5][0-9]|60)\\b", "CISCOTIMESTAMP", "%{MONTH} +%{MONTHDAY} %{YEAR} %{HOUR}:%{MINUTE}:(?:[0-5][0-9]|60)\\b", "CISCOTIMESTAMP",
Arrays.asList(" 11 1111 11 11 11", " 1 1111 11 11 11"), 1, 0), Arrays.asList(" 11 1111 11 11 11", " 1 1111 11 11 11"), 1, 0),
new CandidateTimestampFormat(CandidateTimestampFormat::indeterminateDayMonthFormatFromExample, new CandidateTimestampFormat(CandidateTimestampFormat::indeterminateDayMonthFormatFromExample,
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "\\b%{DATESTAMP}\\b", "DATESTAMP", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "\\b%{DATESTAMP}\\b", "DATESTAMP",
// In DATESTAMP the month may be 1 or 2 digits, but the day must be 2 // In DATESTAMP the month may be 1 or 2 digits, the year 2 or 4, but the day must be 2
Arrays.asList("11 11 1111 11 11 11", "1 11 1111 11 11 11", "11 1 1111 11 11 11"), 0, 10), // Also note the Grok pattern search space is set to start one character before a quick rule-out
// match because we don't want 11 11 11 matching into 1111 11 11 with this pattern
Arrays.asList("11 11 1111 11 11 11", "1 11 1111 11 11 11", "11 1 1111 11 11 11", "11 11 11 11 11 11", "1 11 11 11 11 11",
"11 1 11 11 11 11"), 1, 10),
new CandidateTimestampFormat(CandidateTimestampFormat::indeterminateDayMonthFormatFromExample, new CandidateTimestampFormat(CandidateTimestampFormat::indeterminateDayMonthFormatFromExample,
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "\\b%{DATE}\\b", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "\\b%{DATE}\\b", "DATE",
// In DATE the month may be 1 or 2 digits, but the day must be 2 // In DATE the month may be 1 or 2 digits, the year 2 or 4, but the day must be 2
Arrays.asList("11 11 1111", "11 1 1111", "1 11 1111"), 0, 0), // Also note the Grok pattern search space is set to start one character before a quick rule-out
// match because we don't want 11 11 11 matching into 1111 11 11 with this pattern
Arrays.asList("11 11 1111", "11 1 1111", "1 11 1111", "11 11 11", "11 1 11", "1 11 11"), 1, 0),
UNIX_MS_CANDIDATE_FORMAT, UNIX_MS_CANDIDATE_FORMAT,
UNIX_CANDIDATE_FORMAT, UNIX_CANDIDATE_FORMAT,
TAI64N_CANDIDATE_FORMAT, TAI64N_CANDIDATE_FORMAT,

190
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java
View File

@ -237,6 +237,14 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
assertEquals(5, indeterminateDateNumbers[0]); assertEquals(5, indeterminateDateNumbers[0]);
assertEquals(15, indeterminateDateNumbers[1]); assertEquals(15, indeterminateDateNumbers[1]);
// US with padding, 2 digit year
indeterminateDateNumbers =
TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("05/15/18 16:14:56",
Collections.singletonList("??/??/yy HH:mm:ss"));
assertEquals(2, indeterminateDateNumbers.length);
assertEquals(5, indeterminateDateNumbers[0]);
assertEquals(15, indeterminateDateNumbers[1]);
// US without padding // US without padding
indeterminateDateNumbers = indeterminateDateNumbers =
TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("5/15/2018 16:14:56", TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("5/15/2018 16:14:56",
@ -245,6 +253,14 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
assertEquals(5, indeterminateDateNumbers[0]); assertEquals(5, indeterminateDateNumbers[0]);
assertEquals(15, indeterminateDateNumbers[1]); assertEquals(15, indeterminateDateNumbers[1]);
// US without padding, 2 digit year
indeterminateDateNumbers =
TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("5/15/18 16:14:56",
Collections.singletonList("?/?/yy HH:mm:ss"));
assertEquals(2, indeterminateDateNumbers.length);
assertEquals(5, indeterminateDateNumbers[0]);
assertEquals(15, indeterminateDateNumbers[1]);
// EU with padding // EU with padding
indeterminateDateNumbers = indeterminateDateNumbers =
TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/05/2018 16:14:56", TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/05/2018 16:14:56",
@ -253,6 +269,14 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
assertEquals(15, indeterminateDateNumbers[0]); assertEquals(15, indeterminateDateNumbers[0]);
assertEquals(5, indeterminateDateNumbers[1]); assertEquals(5, indeterminateDateNumbers[1]);
// EU with padding, 2 digit year
indeterminateDateNumbers =
TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/05/18 16:14:56",
Collections.singletonList("??/??/yy HH:mm:ss"));
assertEquals(2, indeterminateDateNumbers.length);
assertEquals(15, indeterminateDateNumbers[0]);
assertEquals(5, indeterminateDateNumbers[1]);
// EU without padding // EU without padding
indeterminateDateNumbers = indeterminateDateNumbers =
TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/5/2018 16:14:56", TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/5/2018 16:14:56",
@ -260,6 +284,14 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
assertEquals(2, indeterminateDateNumbers.length); assertEquals(2, indeterminateDateNumbers.length);
assertEquals(15, indeterminateDateNumbers[0]); assertEquals(15, indeterminateDateNumbers[0]);
assertEquals(5, indeterminateDateNumbers[1]); assertEquals(5, indeterminateDateNumbers[1]);
// EU without padding, 2 digit year
indeterminateDateNumbers =
TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/5/18 16:14:56",
Collections.singletonList("?/?/yy HH:mm:ss"));
assertEquals(2, indeterminateDateNumbers.length);
assertEquals(15, indeterminateDateNumbers[0]);
assertEquals(5, indeterminateDateNumbers[1]);
} }
public void testDeterminiseJavaTimestampFormat() { public void testDeterminiseJavaTimestampFormat() {
@ -293,6 +325,14 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/5/18 16:14:56");
timestampFormatFinder.addSample("06/6/18 17:14:56");
timestampFormatFinder.addSample("07/7/18 18:14:56");
assertTrue(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats()));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("5/05/2018 16:14:56"); timestampFormatFinder.addSample("5/05/2018 16:14:56");
timestampFormatFinder.addSample("6/06/2018 17:14:56"); timestampFormatFinder.addSample("6/06/2018 17:14:56");
timestampFormatFinder.addSample("7/07/2018 18:14:56"); timestampFormatFinder.addSample("7/07/2018 18:14:56");
@ -302,12 +342,28 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("5/05/18 16:14:56");
timestampFormatFinder.addSample("6/06/18 17:14:56");
timestampFormatFinder.addSample("7/07/18 18:14:56");
assertFalse(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats()));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("5/05/2018 16:14:56"); timestampFormatFinder.addSample("5/05/2018 16:14:56");
timestampFormatFinder.addSample("06/6/2018 17:14:56"); timestampFormatFinder.addSample("06/6/2018 17:14:56");
timestampFormatFinder.addSample("7/07/2018 18:14:56"); timestampFormatFinder.addSample("7/07/2018 18:14:56");
// Inconsistent so no decision // Inconsistent so no decision
assertNull(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats())); assertNull(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats()));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("5/05/18 16:14:56");
timestampFormatFinder.addSample("06/6/18 17:14:56");
timestampFormatFinder.addSample("7/07/18 18:14:56");
assertNull(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats()));
} }
public void testGuessIsDayFirstFromMatchesSingleFormat() { public void testGuessIsDayFirstFromMatchesSingleFormat() {
@ -323,6 +379,15 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("05/15/18 17:14:56");
timestampFormatFinder.addSample("05/25/18 18:14:56");
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null));
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
timestampFormatFinder.addSample("15/05/2018 17:14:56"); timestampFormatFinder.addSample("15/05/2018 17:14:56");
timestampFormatFinder.addSample("25/05/2018 18:14:56"); timestampFormatFinder.addSample("25/05/2018 18:14:56");
@ -332,6 +397,15 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("15/05/18 17:14:56");
timestampFormatFinder.addSample("25/05/18 18:14:56");
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null));
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
timestampFormatFinder.addSample("05/06/2018 17:14:56"); timestampFormatFinder.addSample("05/06/2018 17:14:56");
timestampFormatFinder.addSample("05/07/2018 18:14:56"); timestampFormatFinder.addSample("05/07/2018 18:14:56");
@ -340,6 +414,13 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null));
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null));
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("05/06/18 17:14:56");
timestampFormatFinder.addSample("05/07/18 18:14:56");
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null));
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
@ -352,6 +433,15 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("06/05/18 17:14:56");
timestampFormatFinder.addSample("07/05/18 18:14:56");
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null));
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
timestampFormatFinder.addSample("06/06/2018 17:14:56"); timestampFormatFinder.addSample("06/06/2018 17:14:56");
timestampFormatFinder.addSample("07/07/2018 18:14:56"); timestampFormatFinder.addSample("07/07/2018 18:14:56");
@ -359,6 +449,15 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
// Insufficient evidence to decide // Insufficient evidence to decide
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null)); assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null));
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null)); assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("06/06/18 17:14:56");
timestampFormatFinder.addSample("07/07/18 18:14:56");
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null));
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null));
} }
public void testGuessIsDayFirstFromMatchesMultipleFormats() { public void testGuessIsDayFirstFromMatchesMultipleFormats() {
@ -366,9 +465,11 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
// Similar to the test above, but with the possibility that the secondary // Similar to the test above, but with the possibility that the secondary
// ISO8601 formats cause confusion - this test proves that they don't // ISO8601 formats cause confusion - this test proves that they don't
// DATESTAMP supports both 2 and 4 digit years, so each test is repeated for both lengths
TimestampFormatFinder.TimestampFormat expectedPrimaryFormat = TimestampFormatFinder.TimestampFormat expectedPrimaryFormat =
new TimestampFormatFinder.TimestampFormat(Collections.singletonList("??/??/yyyy HH:mm:ss"), new TimestampFormatFinder.TimestampFormat(Collections.singletonList("??/??/yyyy HH:mm:ss"),
Pattern.compile("\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b"), "DATESTAMP", Collections.emptyMap(), ""); Pattern.compile("\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b"), "DATESTAMP",
Collections.emptyMap(), "");
TimestampFormatFinder timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); TimestampFormatFinder timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
@ -383,6 +484,17 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("2018-05-15T17:14:56");
timestampFormatFinder.addSample("05/15/18 17:14:56");
timestampFormatFinder.addSample("2018-05-25T18:14:56");
timestampFormatFinder.addSample("05/25/18 18:14:56");
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
timestampFormatFinder.addSample("2018-05-15T17:14:56"); timestampFormatFinder.addSample("2018-05-15T17:14:56");
timestampFormatFinder.addSample("15/05/2018 17:14:56"); timestampFormatFinder.addSample("15/05/2018 17:14:56");
@ -394,6 +506,17 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("2018-05-15T17:14:56");
timestampFormatFinder.addSample("15/05/18 17:14:56");
timestampFormatFinder.addSample("2018-05-25T18:14:56");
timestampFormatFinder.addSample("25/05/18 18:14:56");
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
timestampFormatFinder.addSample("2018-05-06T17:14:56"); timestampFormatFinder.addSample("2018-05-06T17:14:56");
timestampFormatFinder.addSample("05/06/2018 17:14:56"); timestampFormatFinder.addSample("05/06/2018 17:14:56");
@ -406,6 +529,17 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("2018-05-06T17:14:56");
timestampFormatFinder.addSample("05/06/18 17:14:56");
timestampFormatFinder.addSample("2018-05-07T18:14:56");
timestampFormatFinder.addSample("05/07/18 18:14:56");
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
timestampFormatFinder.addSample("2018-05-06T17:14:56"); timestampFormatFinder.addSample("2018-05-06T17:14:56");
timestampFormatFinder.addSample("06/05/2018 17:14:56"); timestampFormatFinder.addSample("06/05/2018 17:14:56");
@ -418,6 +552,17 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("2018-05-06T17:14:56");
timestampFormatFinder.addSample("06/05/18 17:14:56");
timestampFormatFinder.addSample("2018-05-07T18:14:56");
timestampFormatFinder.addSample("07/05/18 18:14:56");
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/05/2018 16:14:56");
timestampFormatFinder.addSample("2018-06-06T17:14:56"); timestampFormatFinder.addSample("2018-06-06T17:14:56");
timestampFormatFinder.addSample("06/06/2018 17:14:56"); timestampFormatFinder.addSample("06/06/2018 17:14:56");
@ -427,6 +572,17 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
// Insufficient evidence to decide // Insufficient evidence to decide
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER);
timestampFormatFinder.addSample("05/05/18 16:14:56");
timestampFormatFinder.addSample("2018-06-06T17:14:56");
timestampFormatFinder.addSample("06/06/18 17:14:56");
timestampFormatFinder.addSample("2018-07-07T18:14:56");
timestampFormatFinder.addSample("07/07/18 18:14:56");
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat));
} }
public void testGuessIsDayFirstFromLocale() { public void testGuessIsDayFirstFromLocale() {
@ -674,31 +830,31 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
Arrays.asList("MMM dd yyyy HH:mm:ss", "MMM d yyyy HH:mm:ss", "MMM d yyyy HH:mm:ss"), 1526400896000L); Arrays.asList("MMM dd yyyy HH:mm:ss", "MMM d yyyy HH:mm:ss", "MMM d yyyy HH:mm:ss"), 1526400896000L);
validateTimestampMatch("05/15/2018 17:14:56,374", "DATESTAMP", validateTimestampMatch("05/15/2018 17:14:56,374", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss,SSS", 1526400896374L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss,SSS", 1526400896374L);
validateTimestampMatch("05-15-2018-17:14:56.374", "DATESTAMP", validateTimestampMatch("05-15-2018-17:14:56.374", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss.SSS", 1526400896374L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss.SSS", 1526400896374L);
validateTimestampMatch("15/05/2018 17:14:56.374", "DATESTAMP", validateTimestampMatch("15/05/2018 17:14:56.374", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss.SSS", 1526400896374L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss.SSS", 1526400896374L);
validateTimestampMatch("15-05-2018-17:14:56,374", "DATESTAMP", validateTimestampMatch("15-05-2018-17:14:56,374", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss,SSS", 1526400896374L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss,SSS", 1526400896374L);
validateTimestampMatch("15.05.2018 17:14:56.374", "DATESTAMP", validateTimestampMatch("15.05.2018 17:14:56.374", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss.SSS", 1526400896374L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss.SSS", 1526400896374L);
validateTimestampMatch("05/15/2018 17:14:56", "DATESTAMP", validateTimestampMatch("05/15/2018 17:14:56", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss", 1526400896000L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss", 1526400896000L);
validateTimestampMatch("05-15-2018-17:14:56", "DATESTAMP", validateTimestampMatch("05-15-2018-17:14:56", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss", 1526400896000L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss", 1526400896000L);
validateTimestampMatch("15/05/2018 17:14:56", "DATESTAMP", validateTimestampMatch("15/05/2018 17:14:56", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss", 1526400896000L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss", 1526400896000L);
validateTimestampMatch("15-05-2018-17:14:56", "DATESTAMP", validateTimestampMatch("15-05-2018-17:14:56", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss", 1526400896000L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss", 1526400896000L);
validateTimestampMatch("15.05.2018 17:14:56", "DATESTAMP", validateTimestampMatch("15.05.2018 17:14:56", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss", 1526400896000L); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss", 1526400896000L);
validateTimestampMatch("05/15/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "MM/dd/yyyy", 1526338800000L); validateTimestampMatch("05/15/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "MM/dd/yyyy", 1526338800000L);
validateTimestampMatch("05-15-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "MM-dd-yyyy", 1526338800000L); validateTimestampMatch("05-15-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "MM-dd-yyyy", 1526338800000L);
validateTimestampMatch("15/05/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "dd/MM/yyyy", 1526338800000L); validateTimestampMatch("15/05/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "dd/MM/yyyy", 1526338800000L);
validateTimestampMatch("15-05-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "dd-MM-yyyy", 1526338800000L); validateTimestampMatch("15-05-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "dd-MM-yyyy", 1526338800000L);
validateTimestampMatch("15.05.2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "dd.MM.yyyy", 1526338800000L); validateTimestampMatch("15.05.2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "dd.MM.yyyy", 1526338800000L);
} }
public void testFindFormatGivenOnlySystemDate() { public void testFindFormatGivenOnlySystemDate() {
@ -811,7 +967,7 @@ public class TimestampFormatFinderTests extends FileStructureTestCase {
validateFindInFullMessage("10-28-2016 16:22:47.636 +0200 ERROR Network - " + validateFindInFullMessage("10-28-2016 16:22:47.636 +0200 ERROR Network - " +
"Error encountered for connection from src=192.168.0.1:12345. Local side shutting down", "", "DATESTAMP", "Error encountered for connection from src=192.168.0.1:12345. Local side shutting down", "", "DATESTAMP",
"\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy HH:mm:ss.SSS"); "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy HH:mm:ss.SSS");
validateFindInFullMessage("2018-01-06 19:22:20.106822|INFO |VirtualServer |1 |client " + validateFindInFullMessage("2018-01-06 19:22:20.106822|INFO |VirtualServer |1 |client " +
" 'User1'(id:2) was added to channelgroup 'Channel Admin'(id:5) by client 'User1'(id:2) in channel '3er Instanz'(id:2)", "", " 'User1'(id:2) was added to channelgroup 'Channel Admin'(id:5) by client 'User1'(id:2) in channel '3er Instanz'(id:2)", "",