From 424558ff880d9b9489472d37c80fe5bb2fae6fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Lemm=C3=A9?= Date: Wed, 1 May 2019 00:32:19 -0400 Subject: [PATCH] LUCENE-8787: DateRangePrefixTree now parses milliseconds when num digits != 3 --- lucene/CHANGES.txt | 4 +++ .../prefix/tree/DateRangePrefixTree.java | 10 +++---- .../prefix/tree/DateRangePrefixTreeTest.java | 28 +++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 591e647e019..07168135338 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -139,6 +139,10 @@ Improvements of codec internals. This enables a per reader configuration if FSTs are on- or off-heap on a per field basis (Simon Willnauer) +* LUCENE-8787: spatial-extras DateRangePrefixTree used to only parse ISO-8601 timestamps with 0 or 3 + digits of milliseconds precision but now parses other lengths (although > 3 not used). + (Thomas Lemmé via David Smiley) + Changes in Runtime Behavior * LUCENE-8671: Load FST off-heap also for ID-like fields if reader is not opened diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java index 2466b235dae..9ac042810e7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java @@ -504,16 +504,16 @@ public class DateRangePrefixTree extends NumberRangePrefixTree { checkDelimeter(str, offset-1, '.'); //ms: - cal.set(Calendar.MILLISECOND, Integer.parseInt(str.substring(offset, offset+3))); - offset += 3;//last one, move to next char - if (lastOffset == offset) - return cal; + int maxOffset = lastOffset - offset; // assume remaining is all digits to compute milliseconds + // we truncate off > millisecond precision (3 digits only) + int millis = (int) (Integer.parseInt(str.substring(offset, offset + maxOffset)) / Math.pow(10, maxOffset - 3)); + cal.set(Calendar.MILLISECOND, millis); + return cal; } catch (Exception e) { ParseException pe = new ParseException("Improperly formatted datetime: "+str, offset); pe.initCause(e); throw pe; } - throw new ParseException("Improperly formatted datetime: "+str, offset); } private void checkDelimeter(String str, int offset, char delim) { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java index 0670a268c8d..c14026fbd02 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java @@ -17,8 +17,11 @@ package org.apache.lucene.spatial.prefix.tree; import java.text.ParseException; +import java.time.Instant; +import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.temporal.ChronoField; import java.util.Arrays; import java.util.Calendar; import java.util.GregorianCalendar; @@ -30,6 +33,8 @@ import org.apache.lucene.util.LuceneTestCase; import org.locationtech.spatial4j.shape.Shape; import org.locationtech.spatial4j.shape.SpatialRelation; +import static java.time.format.DateTimeFormatter.ISO_DATE_TIME; + public class DateRangePrefixTreeTest extends LuceneTestCase { @ParametersFactory(argumentFormatting = "calendar=%s") @@ -113,6 +118,29 @@ public class DateRangePrefixTreeTest extends LuceneTestCase { assertEquals(cal, tree.parseCalendar(expectedISO8601)); } + public void testParseCalendar() throws ParseException { + Instant expected = OffsetDateTime.of(1984, 12, 18, 12, 34, 56, 100000000, ZoneOffset.UTC).toInstant(); + + assertEquals(expected, tree.parseCalendar("1984-12-18T12:34:56.1Z").toInstant()); + assertEquals(expected.with(ChronoField.MILLI_OF_SECOND, 10), tree.parseCalendar("1984-12-18T12:34:56.01Z").toInstant()); + assertEquals(expected.with(ChronoField.MILLI_OF_SECOND, 1), tree.parseCalendar("1984-12-18T12:34:56.001Z").toInstant()); + assertEquals(expected, tree.parseCalendar("1984-12-18T12:34:56.1000Z").toInstant()); + assertEquals(expected, tree.parseCalendar("1984-12-18T12:34:56.100000000Z").toInstant()); + assertEquals(expected.with(ChronoField.NANO_OF_SECOND, 0), tree.parseCalendar("1984-12-18T12:34:56Z").toInstant()); + // decimal places are simply cut off as rounding may affect the "seconds" part of the calender which was set before + assertEquals(expected.with(ChronoField.MILLI_OF_SECOND, 999), tree.parseCalendar("1984-12-18T12:34:56.9999Z").toInstant()); + + assertEquals(expected, tree.parseCalendar("1984-12-18T12:34:56.1").toInstant()); + assertEquals(expected.with(ChronoField.MILLI_OF_SECOND, 10), tree.parseCalendar("1984-12-18T12:34:56.01").toInstant()); + assertEquals(expected.with(ChronoField.MILLI_OF_SECOND, 1), tree.parseCalendar("1984-12-18T12:34:56.001").toInstant()); + assertEquals(expected, tree.parseCalendar("1984-12-18T12:34:56.1000").toInstant()); + assertEquals(expected, tree.parseCalendar("1984-12-18T12:34:56.100000000").toInstant()); + assertEquals(expected.with(ChronoField.NANO_OF_SECOND, 0), tree.parseCalendar("1984-12-18T12:34:56").toInstant()); + assertEquals(expected.with(ChronoField.MILLI_OF_SECOND, 999), tree.parseCalendar("1984-12-18T12:34:56.9999").toInstant()); + + assertEquals(OffsetDateTime.parse("1984-12-18T12:34:56.01Z", ISO_DATE_TIME).get(ChronoField.MILLI_OF_SECOND), 10); + } + //copies from DateRangePrefixTree private static final int[] CAL_FIELDS = { Calendar.YEAR, Calendar.MONTH, Calendar.DAY_OF_MONTH,