From 82b72ac3fe60babbb1654f705f69eb3e22af6124 Mon Sep 17 00:00:00 2001 From: "Chris M. Hostetter" Date: Fri, 16 Sep 2011 17:42:40 +0000 Subject: [PATCH] SOLR-2772: Fixed Date parsing/formatting of years 0001-1000 git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1171691 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 2 + .../solr/response/TextResponseWriter.java | 55 +---------- .../org/apache/solr/schema/DateField.java | 19 +++- .../apache/solr/BasicFunctionalityTest.java | 14 +++ .../org/apache/solr/schema/DateFieldTest.java | 99 ++++++++++++++++++- .../apache/solr/search/TestRangeQuery.java | 4 +- 6 files changed, 129 insertions(+), 64 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4358217495a..61448ddb539 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -345,6 +345,8 @@ Bug Fixes * SOLR-2726: Fixed NullPointerException when using spellcheck.q with Suggester. (Bernd Fehling, valentin via rmuir) +* SOLR-2772: Fixed Date parsing/formatting of years 0001-1000 (hossman) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java index ea11532c03f..7ebaa8b7f49 100644 --- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java @@ -32,6 +32,7 @@ import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.response.transform.TransformContext; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.DateField; import org.apache.solr.search.DocList; import org.apache.solr.search.ReturnFields; @@ -318,59 +319,7 @@ public abstract class TextResponseWriter { public void writeDate(String name, Date val) throws IOException { - // using a stringBuilder for numbers can be nice since - // a temporary string isn't used (it's added directly to the - // builder's buffer. - - StringBuilder sb = new StringBuilder(); - if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US); - cal.setTime(val); - - int i = cal.get(Calendar.YEAR); - sb.append(i); - sb.append('-'); - i = cal.get(Calendar.MONTH) + 1; // 0 based, so add 1 - if (i<10) sb.append('0'); - sb.append(i); - sb.append('-'); - i=cal.get(Calendar.DAY_OF_MONTH); - if (i<10) sb.append('0'); - sb.append(i); - sb.append('T'); - i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format - if (i<10) sb.append('0'); - sb.append(i); - sb.append(':'); - i=cal.get(Calendar.MINUTE); - if (i<10) sb.append('0'); - sb.append(i); - sb.append(':'); - i=cal.get(Calendar.SECOND); - if (i<10) sb.append('0'); - sb.append(i); - i=cal.get(Calendar.MILLISECOND); - if (i != 0) { - sb.append('.'); - if (i<100) sb.append('0'); - if (i<10) sb.append('0'); - sb.append(i); - - // handle canonical format specifying fractional - // seconds shall not end in '0'. Given the slowness of - // integer div/mod, simply checking the last character - // is probably the fastest way to check. - int lastIdx = sb.length()-1; - if (sb.charAt(lastIdx)=='0') { - lastIdx--; - if (sb.charAt(lastIdx)=='0') { - lastIdx--; - } - sb.setLength(lastIdx+1); - } - - } - sb.append('Z'); - writeDate(name, sb.toString()); + writeDate(name, DateField.formatExternal(val)); } diff --git a/solr/core/src/java/org/apache/solr/schema/DateField.java b/solr/core/src/java/org/apache/solr/schema/DateField.java index 37632fb3850..e923ab87cf3 100644 --- a/solr/core/src/java/org/apache/solr/schema/DateField.java +++ b/solr/core/src/java/org/apache/solr/schema/DateField.java @@ -256,22 +256,29 @@ public class DateField extends FieldType { * Thread safe method that can be used by subclasses to format a Date * using the Internal representation. */ - protected String formatDate(Date d) { + public static String formatDate(Date d) { return fmtThreadLocal.get().format(d); } /** * Return the standard human readable form of the date */ + public static String formatExternal(Date d) { + return fmtThreadLocal.get().format(d) + 'Z'; + } + + /** + * @see {#formatExternal} + */ public String toExternal(Date d) { - return fmtThreadLocal.get().format(d) + 'Z'; + return formatExternal(d); } /** * Thread safe method that can be used by subclasses to parse a Date * that is already in the internal representation */ - protected Date parseDate(String s) throws ParseException { + public static Date parseDate(String s) throws ParseException { return fmtThreadLocal.get().parse(s); } @@ -381,9 +388,13 @@ public class DateField extends FieldType { super.format(d, toAppendTo, pos); /* worry aboutthe milliseconds ourselves */ long millis = d.getTime() % 1000l; - if (0l == millis) { + if (0L == millis) { return toAppendTo; } + if (millis < 0L) { + // original date was prior to epoch + millis += 1000L; + } int posBegin = toAppendTo.length(); toAppendTo.append(millisFormat.format(millis / 1000d)); if (DateFormat.MILLISECOND_FIELD == pos.getField()) { diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index 0e41f731e69..934828bd73b 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -594,6 +594,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { /** @see org.apache.solr.util.DateMathParserTest */ @Test public void testDateMath() { + clearIndex(); // testing everything from query level is hard because // time marches on ... and there is no easy way to reach into the @@ -640,6 +641,19 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { req("q", "bday:[NOW-1MONTH TO NOW+2HOURS]"), "*[count(//doc)=4]"); } + + public void testDateRoundtrip() { + assertU(adoc("id", "99", "bday", "99-01-01T12:34:56.789Z")); + assertU(commit()); + assertQ("year should be canonicallized to 4 digits", + req("q", "id:99"), + "//date[@name='bday'][.='0099-01-01T12:34:56.789Z']"); + assertU(adoc("id", "99", "bday", "1999-01-01T12:34:56.900Z")); + assertU(commit()); + assertQ("millis should be canonicallized to no trailing zeros", + req("q", "id:99"), + "//date[@name='bday'][.='1999-01-01T12:34:56.9Z']"); + } @Test public void testPatternReplaceFilter() { diff --git a/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java b/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java index 0dbfd1c36f7..a34f2f6ae5d 100644 --- a/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java @@ -21,6 +21,9 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.util.DateMathParser; + +import org.junit.Ignore; + import java.util.Date; import java.util.TimeZone; import java.util.Locale; @@ -58,7 +61,7 @@ public class DateFieldTest extends LuceneTestCase { assertToI("1995-12-31T23:59:59", "1995-12-31T23:59:59.000Z"); assertToI("1995-12-31T23:59:59", "1995-12-31T23:59:59.00Z"); assertToI("1995-12-31T23:59:59", "1995-12-31T23:59:59.0Z"); - + // kind of kludgy, but we have other tests for the actual date math assertToI(f.toInternal(p.parseMath("/DAY")), "NOW/DAY"); @@ -100,21 +103,107 @@ public class DateFieldTest extends LuceneTestCase { // as of Solr1.3 public void testToObject() throws Exception { + + // just after epoch + assertToObject( 5L, "1970-01-01T00:00:00.005Z"); + assertToObject( 0L, "1970-01-01T00:00:00Z"); + assertToObject(370L, "1970-01-01T00:00:00.37Z"); + assertToObject(900L, "1970-01-01T00:00:00.9Z"); + + // well after epoch assertToObject(820454399987l, "1995-12-31T23:59:59.987666Z"); assertToObject(820454399987l, "1995-12-31T23:59:59.987Z"); assertToObject(820454399980l, "1995-12-31T23:59:59.98Z"); assertToObject(820454399900l, "1995-12-31T23:59:59.9Z"); assertToObject(820454399000l, "1995-12-31T23:59:59Z"); + + // waaaay after epoch + assertToObject(327434918399005L, "12345-12-31T23:59:59.005Z"); + assertToObject(327434918399000L, "12345-12-31T23:59:59Z"); + assertToObject(327434918399370L, "12345-12-31T23:59:59.37Z"); + assertToObject(327434918399900L, "12345-12-31T23:59:59.9Z"); + + // well before epoch + assertToObject(-52700112001000L, "0299-12-31T23:59:59Z"); + assertToObject(-52700112000877L, "0299-12-31T23:59:59.123Z"); + assertToObject(-52700112000910L, "0299-12-31T23:59:59.09Z"); + + // flexible in parsing years less then 4 digits + assertToObject(-52700112001000L, "299-12-31T23:59:59Z"); + } public void testFormatter() { - assertEquals("1970-01-01T00:00:00.005", f.formatDate(new Date(5))); - assertEquals("1970-01-01T00:00:00", f.formatDate(new Date(0))); - assertEquals("1970-01-01T00:00:00.37", f.formatDate(new Date(370))); - assertEquals("1970-01-01T00:00:00.9", f.formatDate(new Date(900))); + // just after epoch + assertFormat("1970-01-01T00:00:00.005", 5L); + assertFormat("1970-01-01T00:00:00", 0L); + assertFormat("1970-01-01T00:00:00.37", 370L); + assertFormat("1970-01-01T00:00:00.9", 900L); + + // well after epoch + assertFormat("1999-12-31T23:59:59.005", 946684799005L); + assertFormat("1999-12-31T23:59:59", 946684799000L); + assertFormat("1999-12-31T23:59:59.37", 946684799370L); + assertFormat("1999-12-31T23:59:59.9", 946684799900L); + + // waaaay after epoch + assertFormat("12345-12-31T23:59:59.005", 327434918399005L); + assertFormat("12345-12-31T23:59:59", 327434918399000L); + assertFormat("12345-12-31T23:59:59.37", 327434918399370L); + assertFormat("12345-12-31T23:59:59.9", 327434918399900L); + + // well before epoch + assertFormat("0299-12-31T23:59:59", -52700112001000L); + assertFormat("0299-12-31T23:59:59.123", -52700112000877L); + assertFormat("0299-12-31T23:59:59.09", -52700112000910L); } + /** + * Using dates in the canonical format, verify that parsing+formating + * is an identify function + */ + public void testRoundTrip() throws Exception { + + // typical dates, various precision + assertRoundTrip("1995-12-31T23:59:59.987Z"); + assertRoundTrip("1995-12-31T23:59:59.98Z"); + assertRoundTrip("1995-12-31T23:59:59.9Z"); + assertRoundTrip("1995-12-31T23:59:59Z"); + assertRoundTrip("1976-03-06T03:06:00Z"); + + // dates with atypical years + assertRoundTrip("0001-01-01T01:01:01Z"); + assertRoundTrip("12021-12-01T03:03:03Z"); + } + + @Ignore("SOLR-2773: Non-Positive years don't work") + public void testRoundTripNonPositiveYear() throws Exception { + + // :TODO: ambiguity about year zero + // assertRoundTrip("0000-04-04T04:04:04Z"); + + // dates with negative years + assertRoundTrip("-0005-05-05T05:05:05Z"); + assertRoundTrip("-2021-12-01T04:04:04Z"); + assertRoundTrip("-12021-12-01T02:02:02Z"); + + // :TODO: assertFormat and assertToObject some negative years + + } + + protected void assertFormat(final String expected, final long millis) { + assertEquals(expected, f.formatDate(new Date(millis))); + } + + protected void assertRoundTrip(String canonicalDate) throws Exception { + Date d = DateField.parseDate(canonicalDate); + String result = DateField.formatDate(d) + "Z"; + assertEquals("d:" + d.getTime(), canonicalDate, result); + + } + + public void testCreateField() { int props = FieldProperties.INDEXED ^ FieldProperties.STORED; SchemaField sf = new SchemaField( "test", f, props, null ); diff --git a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java index 1a1f00b1cd6..8aa85ded418 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java @@ -85,7 +85,7 @@ public class TestRangeQuery extends SolrTestCaseJ4 { String[] longs = {""+(l-1), ""+(l), ""+(l+1), ""+(l-2), ""+(l+2)}; String[] doubles = {""+(d-1e-16), ""+(d), ""+(d+1e-16), ""+(d-2e-16), ""+(d+2e-16)}; String[] strings = {"aaa","bbb","ccc", "aa","cccc" }; - String[] dates = {"1999-12-31T23:59:59.999Z","2000-01-01T00:00:00.000Z","2000-01-01T00:00:00.001Z", "1999-12-31T23:59:59.998Z","2000-01-01T00:00:00.002Z" }; + String[] dates = {"0299-12-31T23:59:59.999Z","2000-01-01T00:00:00.000Z","2000-01-01T00:00:00.001Z", "0299-12-31T23:59:59.998Z","2000-01-01T00:00:00.002Z" }; // fields that normal range queries should work on Map norm_fields = new HashMap(); @@ -285,4 +285,4 @@ public class TestRangeQuery extends SolrTestCaseJ4 { } return true; } -} \ No newline at end of file +}