SOLR-1474 -- Delta-import should run even if last_index_time is not set

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@820235 13f79535-47bb-0310-9956-ffa450edef68

contrib/dataimporthandler/CHANGES.txt

@@ -29,6 +29,10 @@ from Transformers into an EntityProcessorWrapper class. The EntityProcessor#dest
 parent-row at the end of row (end of data). A new method EntityProcessor#close is added which is called at the end
 of import.
 
+In Solr 1.3, if the last_index_time was not available (first import) and a delta-import was requested, a full-import
+was run instead. This is no longer the case. In Solr 1.4 delta import is run with last_index_time as the epoch
+date (January 1, 1970, 00:00:00 GMT) if last_index_time is not available.
+
 Detailed Change List
 ----------------------
 
@@ -265,6 +269,9 @@ Bug Fixes
 30.SOLR-1450: Jdbc connection properties such as batchSize are not applied if the driver jar is placed
               in solr_home/lib.
               (Steve Sun via shalin)
+
+31.SOLR-1474: Delta-import should run even if last_index_time is not set.
+              (shalin)
              
 
 Documentation

contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java

@@ -22,6 +22,7 @@ import org.apache.solr.util.AbstractSolrTestCase;
 import org.apache.solr.common.util.NamedList;
 
 import java.io.IOException;
+import java.io.File;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -46,6 +47,14 @@ public abstract class AbstractDataImportHandlerTest extends
 
   @Override
   public void tearDown() throws Exception {
+    // remove dataimport.properties
+    File f = new File("solr/conf/dataimport.properties");
+    log.info("Looking for dataimport.properties at: " + f.getAbsolutePath());
+    if (f.exists()) {
+      log.info("Deleting dataimport.properties");
+      if (!f.delete())
+        log.warn("Could not delete dataimport.properties");
+    }
     super.tearDown();
   }
 

contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java

@@ -252,14 +252,6 @@ public class DataImporter {
     this.indexStartTime = indextStartTime;
   }
 
-  Date getLastIndexTime() {
-    return lastIndexTime;
-  }
-
-  void setLastIndexTime(Date lastIndexTime) {
-    this.lastIndexTime = lastIndexTime;
-  }
-
   void store(Object key, Object value) {
     store.put(key, value);
   }
@@ -387,8 +379,6 @@ public class DataImporter {
       return;
     }
     try {
-      Date lastModified = sw.loadIndexStartTime();
-      setLastIndexTime(lastModified);
       if (FULL_IMPORT_CMD.equals(command) || IMPORT_CMD.equals(command)) {
         doFullImport(sw, reqParams);
       } else if (command.equals(DELTA_IMPORT_CMD)) {

contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java

@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import java.text.ParseException;
 
 /**
  * <p> DocBuilder is responsible for creating Solr documents out of the given configuration. It also maintains
@@ -41,6 +42,8 @@ public class DocBuilder {
 
   private static final Logger LOG = LoggerFactory.getLogger(DocBuilder.class);
 
+  private static final Date EPOCH = new Date(0);
+
   DataImporter dataImporter;
 
   private DataConfig.Document document;
@@ -76,24 +79,36 @@ public class DocBuilder {
   }
 
   public VariableResolverImpl getVariableResolver() {
-    VariableResolverImpl resolver = new VariableResolverImpl();
-    Map<String, Object> indexerNamespace = new HashMap<String, Object>();
-    if (dataImporter.getLastIndexTime() != null)
-      indexerNamespace.put(LAST_INDEX_TIME,
-              DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getLastIndexTime()));
-    indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
-    indexerNamespace.put("request", requestParameters.requestParams);
-    indexerNamespace.put("functions", functionsNamespace);
-    for (DataConfig.Entity entity : dataImporter.getConfig().document.entities) {
-      String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY;
-      String lastIndex = persistedProperties.getProperty(key);
-      if (lastIndex != null) {
-        indexerNamespace.put(key, lastIndex);
+    try {
+      VariableResolverImpl resolver = new VariableResolverImpl();
+      Map<String, Object> indexerNamespace = new HashMap<String, Object>();
+      if (persistedProperties.getProperty(LAST_INDEX_TIME) != null) {
+        indexerNamespace.put(LAST_INDEX_TIME,
+                DataImporter.DATE_TIME_FORMAT.get().parse(persistedProperties.getProperty(LAST_INDEX_TIME)));
+      } else  {
+        // set epoch
+        indexerNamespace.put(LAST_INDEX_TIME, EPOCH);
       }
+      indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
+      indexerNamespace.put("request", requestParameters.requestParams);
+      indexerNamespace.put("functions", functionsNamespace);
+      for (DataConfig.Entity entity : dataImporter.getConfig().document.entities) {
+        String key = entity.name + "." + SolrWriter.LAST_INDEX_KEY;
+        String lastIndex = persistedProperties.getProperty(key);
+        if (lastIndex != null) {
+          indexerNamespace.put(key, lastIndex);
+        } else  {
+          indexerNamespace.put(key, EPOCH);
+        }
+      }
+      resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace);
+      resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
+      return resolver;
+    } catch (ParseException e) {
+      DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e);
+      // unreachable statement
+      return null;
     }
-    resolver.addNamespace(DataConfig.IMPORTER_NS_SHORT, indexerNamespace);
-    resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
-    return resolver;
   }
 
   private void invokeEventListener(String className) {
@@ -153,8 +168,7 @@ public class DocBuilder {
               DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
       root = e;
       String delQuery = e.allAttributes.get("preImportDeleteQuery");
-      if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP
-              && dataImporter.getLastIndexTime() != null) {
+      if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
         cleanByQuery(delQuery, fullCleanDone);
         doDelta();
         delQuery = e.allAttributes.get("postImportDeleteQuery");

contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java

@@ -219,22 +219,6 @@ public class SolrWriter {
     }
   }
 
-  public Date loadIndexStartTime() {
-    Properties props;
-    props = readIndexerProperties();
-    String result = props.getProperty(SolrWriter.LAST_INDEX_KEY);
-
-    try {
-      if (result != null)
-        return DataImporter.DATE_TIME_FORMAT.get().parse(result);
-    } catch (ParseException e) {
-      throw new DataImportHandlerException(DataImportHandlerException.WARN,
-              "Unable to read last indexed time from: "
-                      + persistFilename, e);
-    }
-    return null;
-  }
-
   public DebugLogger getDebugLogger() {
     if (debugLogger == null) {
       debugLogger = new DebugLogger(this);