SOLR-1141 -- NullPointerException during snapshoot command in java based replication

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@771378 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -359,6 +359,8 @@ Bug Fixes
 43. SOLR-929: LukeRequestHandler should return "dynamicBase" only if the field is dynamic.
     (Peter Wolanin, koji)
 
+44. SOLR-1141: NullPointerException during snapshoot command in java based replication (Jian Han Guo, shalin)
+
 Other Changes
 ----------------------
  1. Upgraded to Lucene 2.4.0 (yonik)

src/java/org/apache/solr/handler/ReplicationHandler.java

@@ -34,6 +34,7 @@ import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.update.DirectUpdateHandler2;
 import org.apache.solr.util.RefCounted;
 import org.apache.solr.util.plugin.SolrCoreAware;
+import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -92,6 +93,8 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
 
   private volatile IndexCommit indexCommitPoint;
 
+  volatile NamedList snapShootDetails; 
+
   public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
     rsp.setHttpCaching(false);
     final SolrParams solrParams = req.getParams();
@@ -201,7 +204,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
     } catch (Exception e) {
       LOG.warn("Exception in finding checksum of " + f, e);
     } finally {
-      closeNoExp(fis);
+      IOUtils.closeQuietly(fis);
     }
     return null;
   }
@@ -236,8 +239,12 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
 
   private void doSnapShoot(SolrQueryResponse rsp) {
     try {
-      new SnapShooter(core).createSnapAsync(core.getDeletionPolicy().getLatestCommit().getFileNames());
+      IndexCommit indexCommit = core.getDeletionPolicy().getLatestCommit();
+      if (indexCommit != null)  {
+        new SnapShooter(core).createSnapAsync(indexCommit.getFileNames(), this);
+      }
     } catch (Exception e) {
+      LOG.warn("Exception during creating a snapshot", e);
       rsp.add("exception", e);
     }
   }
@@ -542,7 +549,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
       } catch (Exception e) {
         LOG.warn("Exception while reading " + SnapPuller.REPLICATION_PROPERTIES);
       } finally {
-        closeNoExp(inFile);
+        IOUtils.closeQuietly(inFile);
       }
       try {
         NamedList nl = snapPuller.getCommandResponse(CMD_DETAILS);
@@ -667,6 +674,9 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
       details.add("master", master);
     if(isSlave)
       details.add("slave", slave);
+    NamedList snapshotStats = snapShootDetails;
+    if (snapshotStats != null)
+      details.add(CMD_SNAP_SHOOT, snapshotStats);
     return details;
   }
 
@@ -804,7 +814,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
         if (snapshoot) {
           try {
             SnapShooter snapShooter = new SnapShooter(core);
-            snapShooter.createSnapAsync(core.getDeletionPolicy().getLatestCommit().getFileNames());
+            snapShooter.createSnapAsync(core.getDeletionPolicy().getLatestCommit().getFileNames(), ReplicationHandler.this);
           } catch (Exception e) {
             LOG.error("Exception while snapshooting", e);
           }
@@ -815,13 +825,6 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
     };
   }
 
-  static void closeNoExp(Closeable closeable) {
-    try {
-      if (closeable != null)
-        closeable.close();
-    } catch (Exception e) {/*no op*/ }
-  }
-
   private class FileStream {
     private SolrParams params;
 
@@ -913,7 +916,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
       } catch (IOException e) {
         LOG.warn("Exception while writing response for params: " + params, e);
       } finally {
-        closeNoExp(inputStream);
+        IOUtils.closeQuietly(inputStream);
       }
     }
 

src/java/org/apache/solr/handler/SnapPuller.java

@@ -18,6 +18,7 @@ package org.apache.solr.handler;
 
 import org.apache.commons.httpclient.*;
 import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.io.IOUtils;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.util.FastInputStream;
@@ -380,8 +381,8 @@ public class SnapPuller {
       LOG.warn("Exception while updating statistics", e);
     }
     finally {
-      closeNoExp(inFile);
+      IOUtils.closeQuietly(inFile);
-      closeNoExp(outFile);
+      IOUtils.closeQuietly(outFile);
     }
   }
 
@@ -584,7 +585,7 @@ public class SnapPuller {
       } catch (Exception e) {
         LOG.error("Unable to load index.properties");
       } finally {
-        closeNoExp(is);
+        IOUtils.closeQuietly(is);
       }
     }
     p.put("index", snap);
@@ -596,7 +597,7 @@ public class SnapPuller {
       throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
               "Unable to write index.properties", e);
     } finally {
-      closeNoExp(os);
+      IOUtils.closeQuietly(os);
     }
   }
 
@@ -818,7 +819,7 @@ public class SnapPuller {
             }
             //if there is an error continue. But continue from the point where it got broken
           } finally {
-            closeNoExp(is);
+            IOUtils.closeQuietly(is);
           }
         }
       } finally {

src/java/org/apache/solr/handler/SnapShooter.java

@@ -16,86 +16,85 @@
  */
 package org.apache.solr.handler;
 
-import org.apache.solr.common.SolrException;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.SimpleFSLockFactory;
 import org.apache.solr.core.SolrCore;
+import org.apache.solr.common.util.NamedList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.Collection;
 import java.util.Date;
-import java.util.WeakHashMap;
+import java.util.ArrayList;
 
 /**
- * <p/>
+ * <p/> Provides functionality equivalent to the snapshooter script </p>
- * Provides functionality equivalent to the snapshooter script
- * </p>
  *
  * @version $Id$
  * @since solr 1.4
  */
 public class SnapShooter {
+  private static final Logger LOG = LoggerFactory.getLogger(SnapShooter.class.getName());
   private String snapDir = null;
   private SolrCore solrCore;
+  private SimpleFSLockFactory lockFactory;
 
-  public SnapShooter(SolrCore core) {
+  public SnapShooter(SolrCore core) throws IOException {
     solrCore = core;
     snapDir = core.getDataDir();
+    lockFactory = new SimpleFSLockFactory(snapDir);
   }
 
-  void createSnapAsync(final Collection<String> files) {
+  void createSnapAsync(final Collection<String> files, final ReplicationHandler replicationHandler) {
     new Thread() {
       public void run() {
-        createSnapshot(files);
+        createSnapshot(files, replicationHandler);
       }
     }.start();
   }
 
-  void createSnapshot(Collection<String> files) {
+  void createSnapshot(Collection<String> files, ReplicationHandler replicationHandler) {
-    File lockFile = null;
+    NamedList details = new NamedList();
+    details.add("startTime", new Date().toString());
     File snapShotDir = null;
     String directoryName = null;
+    Lock lock = null;
     try {
-      lockFile = new File(snapDir, directoryName + ".lock");
-      if (lockFile.exists()) {
-        return;
-      }
       SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT);
       directoryName = "snapshot." + fmt.format(new Date());
+      lock = lockFactory.makeLock(directoryName + ".lock");
+      if (lock.isLocked()) return;
       snapShotDir = new File(snapDir, directoryName);
-      lockFile.createNewFile();
+      if (!snapShotDir.mkdir()) {
-      snapShotDir.mkdir();
+        LOG.warn("Unable to create snapshot directory: " + snapShotDir.getAbsolutePath());
-      for (String indexFile : files) {
+        return;
-        copyFile2Dir(new File(solrCore.getIndexDir(), indexFile), snapShotDir);
       }
+      for (String indexFile : files) {
+        FileUtils.copyFileToDirectory(new File(solrCore.getIndexDir(), indexFile), snapShotDir, true);
+      }
+      details.add("fileCount", files.size());
+      details.add("status", "success");
+      details.add("snapshotCompletedAt", new Date().toString());
     } catch (Exception e) {
       SnapPuller.delTree(snapShotDir);
-      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+      LOG.error("Exception while creating snapshot", e);
+      details.add("snapShootException", e.getMessage());
     } finally {
-      if (lockFile != null) {
+      replicationHandler.snapShootDetails = details;
-        lockFile.delete();
+      if (lock != null) {
+        try {
+          lock.release();
+        } catch (IOException e) {
+          LOG.error("Unable to release snapshoot lock: " + directoryName + ".lock");
+        }
       }
     }
   }
 
-  static void copyFile2Dir(File file, File toDir) throws IOException {
-    FileInputStream fis = null;
-    FileOutputStream fos = null;
-    try {
-      fis = new FileInputStream(file);
-      File destFile = new File(toDir, file.getName());
-      fos = new FileOutputStream(destFile);
-      fis.getChannel().transferTo(0, fis.available(), fos.getChannel());
-      destFile.setLastModified(file.lastModified());
-    } finally {
-      ReplicationHandler.closeNoExp(fis);
-      ReplicationHandler.closeNoExp(fos);
-    }
-  }
-
   public static final String SNAP_DIR = "snapDir";
   public static final String DATE_FMT = "yyyyMMddhhmmss";
-  private static WeakHashMap<SolrCore, SnapShooter> SNAP_DIRS = new WeakHashMap<SolrCore, SnapShooter>();
 }