SOLR-2772: Fixed Date parsing/formatting of years 0001-1000

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1171691 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -345,6 +345,8 @@ Bug Fixes
 * SOLR-2726: Fixed NullPointerException when using spellcheck.q with Suggester.
   (Bernd Fehling, valentin via rmuir)
 
+* SOLR-2772: Fixed Date parsing/formatting of years 0001-1000 (hossman)
+
  Other Changes
 ----------------------
 

solr/core/src/java/org/apache/solr/response/TextResponseWriter.java

@@ -32,6 +32,7 @@ import org.apache.solr.response.transform.DocTransformer;
 import org.apache.solr.response.transform.TransformContext;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.DateField;
 import org.apache.solr.search.DocList;
 import org.apache.solr.search.ReturnFields;
 
@@ -318,59 +319,7 @@ public abstract class TextResponseWriter {
 
 
   public void writeDate(String name, Date val) throws IOException {
-    // using a stringBuilder for numbers can be nice since
+    writeDate(name, DateField.formatExternal(val));
-    // a temporary string isn't used (it's added directly to the
-    // builder's buffer.
-
-    StringBuilder sb = new StringBuilder();
-    if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US);
-    cal.setTime(val);
-
-    int i = cal.get(Calendar.YEAR);
-    sb.append(i);
-    sb.append('-');
-    i = cal.get(Calendar.MONTH) + 1;  // 0 based, so add 1
-    if (i<10) sb.append('0');
-    sb.append(i);
-    sb.append('-');
-    i=cal.get(Calendar.DAY_OF_MONTH);
-    if (i<10) sb.append('0');
-    sb.append(i);
-    sb.append('T');
-    i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format
-    if (i<10) sb.append('0');
-    sb.append(i);
-    sb.append(':');
-    i=cal.get(Calendar.MINUTE);
-    if (i<10) sb.append('0');
-    sb.append(i);
-    sb.append(':');
-    i=cal.get(Calendar.SECOND);
-    if (i<10) sb.append('0');
-    sb.append(i);
-    i=cal.get(Calendar.MILLISECOND);
-    if (i != 0) {
-      sb.append('.');
-      if (i<100) sb.append('0');
-      if (i<10) sb.append('0');
-      sb.append(i);
-
-      // handle canonical format specifying fractional
-      // seconds shall not end in '0'.  Given the slowness of
-      // integer div/mod, simply checking the last character
-      // is probably the fastest way to check.
-      int lastIdx = sb.length()-1;
-      if (sb.charAt(lastIdx)=='0') {
-        lastIdx--;
-        if (sb.charAt(lastIdx)=='0') {
-          lastIdx--;
-        }
-        sb.setLength(lastIdx+1);
-      }
-
-    }
-    sb.append('Z');
-    writeDate(name, sb.toString());
   }
   
 

solr/core/src/java/org/apache/solr/schema/DateField.java

@@ -256,22 +256,29 @@ public class DateField extends FieldType {
    * Thread safe method that can be used by subclasses to format a Date
    * using the Internal representation.
    */
-  protected String formatDate(Date d) {
+  public static String formatDate(Date d) {
     return fmtThreadLocal.get().format(d);
   }
 
   /**
    * Return the standard human readable form of the date
    */
+  public static String formatExternal(Date d) {
+    return fmtThreadLocal.get().format(d) + 'Z';
+  }
+
+  /**
+   * @see {#formatExternal}
+   */
   public String toExternal(Date d) {
-    return fmtThreadLocal.get().format(d) + 'Z';  
+    return formatExternal(d);
   }
 
   /**
    * Thread safe method that can be used by subclasses to parse a Date
    * that is already in the internal representation
    */
-   protected Date parseDate(String s) throws ParseException {
+   public static Date parseDate(String s) throws ParseException {
      return fmtThreadLocal.get().parse(s);
    }
 
@@ -381,9 +388,13 @@ public class DateField extends FieldType {
       super.format(d, toAppendTo, pos);
       /* worry aboutthe milliseconds ourselves */
       long millis = d.getTime() % 1000l;
-      if (0l == millis) {
+      if (0L == millis) {
         return toAppendTo;
       }
+      if (millis < 0L) {
+        // original date was prior to epoch
+        millis += 1000L;
+      }
       int posBegin = toAppendTo.length();
       toAppendTo.append(millisFormat.format(millis / 1000d));
       if (DateFormat.MILLISECOND_FIELD == pos.getField()) {

solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java

@@ -594,6 +594,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
   /** @see org.apache.solr.util.DateMathParserTest */
   @Test
   public void testDateMath() {
+    clearIndex();
 
     // testing everything from query level is hard because
     // time marches on ... and there is no easy way to reach into the
@@ -640,6 +641,19 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
             req("q", "bday:[NOW-1MONTH TO NOW+2HOURS]"), "*[count(//doc)=4]");
     
   }
+
+  public void testDateRoundtrip() {
+    assertU(adoc("id", "99",  "bday", "99-01-01T12:34:56.789Z"));
+    assertU(commit());
+    assertQ("year should be canonicallized to 4 digits",
+            req("q", "id:99"),
+            "//date[@name='bday'][.='0099-01-01T12:34:56.789Z']");
+    assertU(adoc("id", "99",  "bday", "1999-01-01T12:34:56.900Z"));
+    assertU(commit());
+    assertQ("millis should be canonicallized to no trailing zeros",
+            req("q", "id:99"),
+            "//date[@name='bday'][.='1999-01-01T12:34:56.9Z']");
+  }
   
   @Test
   public void testPatternReplaceFilter() {

solr/core/src/test/org/apache/solr/schema/DateFieldTest.java

@@ -21,6 +21,9 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.util.DateMathParser;
+
+import org.junit.Ignore;
+
 import java.util.Date;
 import java.util.TimeZone;
 import java.util.Locale;
@@ -58,7 +61,7 @@ public class DateFieldTest extends LuceneTestCase {
     assertToI("1995-12-31T23:59:59",     "1995-12-31T23:59:59.000Z");
     assertToI("1995-12-31T23:59:59",     "1995-12-31T23:59:59.00Z");
     assertToI("1995-12-31T23:59:59",     "1995-12-31T23:59:59.0Z");
-    
+
     // kind of kludgy, but we have other tests for the actual date math
     assertToI(f.toInternal(p.parseMath("/DAY")), "NOW/DAY");
 
@@ -100,21 +103,107 @@ public class DateFieldTest extends LuceneTestCase {
   
   // as of Solr1.3
   public void testToObject() throws Exception {
+
+    // just after epoch
+    assertToObject(  5L, "1970-01-01T00:00:00.005Z");
+    assertToObject(  0L, "1970-01-01T00:00:00Z");
+    assertToObject(370L, "1970-01-01T00:00:00.37Z");
+    assertToObject(900L, "1970-01-01T00:00:00.9Z");
+
+    // well after epoch
     assertToObject(820454399987l, "1995-12-31T23:59:59.987666Z");
     assertToObject(820454399987l, "1995-12-31T23:59:59.987Z");
     assertToObject(820454399980l, "1995-12-31T23:59:59.98Z");
     assertToObject(820454399900l, "1995-12-31T23:59:59.9Z");
     assertToObject(820454399000l, "1995-12-31T23:59:59Z");
+
+    // waaaay after epoch
+    assertToObject(327434918399005L, "12345-12-31T23:59:59.005Z");
+    assertToObject(327434918399000L, "12345-12-31T23:59:59Z");
+    assertToObject(327434918399370L, "12345-12-31T23:59:59.37Z");
+    assertToObject(327434918399900L, "12345-12-31T23:59:59.9Z");
+
+    // well before epoch
+    assertToObject(-52700112001000L, "0299-12-31T23:59:59Z");
+    assertToObject(-52700112000877L, "0299-12-31T23:59:59.123Z");
+    assertToObject(-52700112000910L, "0299-12-31T23:59:59.09Z");
+
+    // flexible in parsing years less then 4 digits
+    assertToObject(-52700112001000L,  "299-12-31T23:59:59Z");
+
   }
   
   public void testFormatter() {
-    assertEquals("1970-01-01T00:00:00.005", f.formatDate(new Date(5)));
+    // just after epoch
-    assertEquals("1970-01-01T00:00:00",     f.formatDate(new Date(0)));
+    assertFormat("1970-01-01T00:00:00.005", 5L);
-    assertEquals("1970-01-01T00:00:00.37",  f.formatDate(new Date(370)));
+    assertFormat("1970-01-01T00:00:00",     0L);
-    assertEquals("1970-01-01T00:00:00.9",   f.formatDate(new Date(900)));
+    assertFormat("1970-01-01T00:00:00.37",  370L);
+    assertFormat("1970-01-01T00:00:00.9",   900L);
+
+    // well after epoch
+    assertFormat("1999-12-31T23:59:59.005", 946684799005L);
+    assertFormat("1999-12-31T23:59:59",     946684799000L);
+    assertFormat("1999-12-31T23:59:59.37",  946684799370L);
+    assertFormat("1999-12-31T23:59:59.9",   946684799900L);
+
+    // waaaay after epoch
+    assertFormat("12345-12-31T23:59:59.005", 327434918399005L);
+    assertFormat("12345-12-31T23:59:59",     327434918399000L);
+    assertFormat("12345-12-31T23:59:59.37",  327434918399370L);
+    assertFormat("12345-12-31T23:59:59.9",   327434918399900L);
+
+    // well before epoch
+    assertFormat("0299-12-31T23:59:59",     -52700112001000L);
+    assertFormat("0299-12-31T23:59:59.123", -52700112000877L);
+    assertFormat("0299-12-31T23:59:59.09",  -52700112000910L);
 
   }
 
+  /** 
+   * Using dates in the canonical format, verify that parsing+formating 
+   * is an identify function
+   */
+  public void testRoundTrip() throws Exception {
+
+    // typical dates, various precision
+    assertRoundTrip("1995-12-31T23:59:59.987Z");
+    assertRoundTrip("1995-12-31T23:59:59.98Z");
+    assertRoundTrip("1995-12-31T23:59:59.9Z");
+    assertRoundTrip("1995-12-31T23:59:59Z");
+    assertRoundTrip("1976-03-06T03:06:00Z");
+
+    // dates with atypical years
+    assertRoundTrip("0001-01-01T01:01:01Z");
+    assertRoundTrip("12021-12-01T03:03:03Z");
+  }
+
+  @Ignore("SOLR-2773: Non-Positive years don't work")
+  public void testRoundTripNonPositiveYear() throws Exception {
+
+    // :TODO: ambiguity about year zero
+    // assertRoundTrip("0000-04-04T04:04:04Z");
+    
+    // dates with negative years
+    assertRoundTrip("-0005-05-05T05:05:05Z");
+    assertRoundTrip("-2021-12-01T04:04:04Z");
+    assertRoundTrip("-12021-12-01T02:02:02Z");
+    
+    // :TODO: assertFormat and assertToObject some negative years
+
+  }
+
+  protected void assertFormat(final String expected, final long millis) {
+    assertEquals(expected, f.formatDate(new Date(millis)));
+  }
+
+  protected void assertRoundTrip(String canonicalDate) throws Exception {
+    Date d = DateField.parseDate(canonicalDate);
+    String result = DateField.formatDate(d) + "Z";
+    assertEquals("d:" + d.getTime(), canonicalDate, result);
+
+  }
+
+
   public void testCreateField() {
     int props = FieldProperties.INDEXED ^ FieldProperties.STORED;
     SchemaField sf = new SchemaField( "test", f, props, null );

solr/core/src/test/org/apache/solr/search/TestRangeQuery.java

@@ -85,7 +85,7 @@ public class TestRangeQuery extends SolrTestCaseJ4 {
     String[] longs = {""+(l-1), ""+(l), ""+(l+1),  ""+(l-2), ""+(l+2)};
     String[] doubles = {""+(d-1e-16), ""+(d), ""+(d+1e-16),   ""+(d-2e-16), ""+(d+2e-16)};
     String[] strings = {"aaa","bbb","ccc",  "aa","cccc" };
-    String[] dates = {"1999-12-31T23:59:59.999Z","2000-01-01T00:00:00.000Z","2000-01-01T00:00:00.001Z",  "1999-12-31T23:59:59.998Z","2000-01-01T00:00:00.002Z" };
+    String[] dates = {"0299-12-31T23:59:59.999Z","2000-01-01T00:00:00.000Z","2000-01-01T00:00:00.001Z",  "0299-12-31T23:59:59.998Z","2000-01-01T00:00:00.002Z" };
 
     // fields that normal range queries should work on
     Map<String,String[]> norm_fields = new HashMap<String,String[]>();
@@ -285,4 +285,4 @@ public class TestRangeQuery extends SolrTestCaseJ4 {
     }
     return true;
   }
-}
+}