diff --git a/CHANGES.txt b/CHANGES.txt index 15481d16f08..d1385bcec50 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -359,6 +359,8 @@ Bug Fixes 43. SOLR-929: LukeRequestHandler should return "dynamicBase" only if the field is dynamic. (Peter Wolanin, koji) +44. SOLR-1141: NullPointerException during snapshoot command in java based replication (Jian Han Guo, shalin) + Other Changes ---------------------- 1. Upgraded to Lucene 2.4.0 (yonik) diff --git a/src/java/org/apache/solr/handler/ReplicationHandler.java b/src/java/org/apache/solr/handler/ReplicationHandler.java index 90f8cf15816..82de402c491 100644 --- a/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -34,6 +34,7 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.update.DirectUpdateHandler2; import org.apache.solr.util.RefCounted; import org.apache.solr.util.plugin.SolrCoreAware; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,6 +93,8 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw private volatile IndexCommit indexCommitPoint; + volatile NamedList snapShootDetails; + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { rsp.setHttpCaching(false); final SolrParams solrParams = req.getParams(); @@ -201,7 +204,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } catch (Exception e) { LOG.warn("Exception in finding checksum of " + f, e); } finally { - closeNoExp(fis); + IOUtils.closeQuietly(fis); } return null; } @@ -236,8 +239,12 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw private void doSnapShoot(SolrQueryResponse rsp) { try { - new SnapShooter(core).createSnapAsync(core.getDeletionPolicy().getLatestCommit().getFileNames()); + IndexCommit indexCommit = core.getDeletionPolicy().getLatestCommit(); + if (indexCommit != null) { + new SnapShooter(core).createSnapAsync(indexCommit.getFileNames(), this); + } } catch (Exception e) { + LOG.warn("Exception during creating a snapshot", e); rsp.add("exception", e); } } @@ -542,7 +549,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } catch (Exception e) { LOG.warn("Exception while reading " + SnapPuller.REPLICATION_PROPERTIES); } finally { - closeNoExp(inFile); + IOUtils.closeQuietly(inFile); } try { NamedList nl = snapPuller.getCommandResponse(CMD_DETAILS); @@ -667,6 +674,9 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw details.add("master", master); if(isSlave) details.add("slave", slave); + NamedList snapshotStats = snapShootDetails; + if (snapshotStats != null) + details.add(CMD_SNAP_SHOOT, snapshotStats); return details; } @@ -804,7 +814,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw if (snapshoot) { try { SnapShooter snapShooter = new SnapShooter(core); - snapShooter.createSnapAsync(core.getDeletionPolicy().getLatestCommit().getFileNames()); + snapShooter.createSnapAsync(core.getDeletionPolicy().getLatestCommit().getFileNames(), ReplicationHandler.this); } catch (Exception e) { LOG.error("Exception while snapshooting", e); } @@ -815,13 +825,6 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw }; } - static void closeNoExp(Closeable closeable) { - try { - if (closeable != null) - closeable.close(); - } catch (Exception e) {/*no op*/ } - } - private class FileStream { private SolrParams params; @@ -913,7 +916,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } catch (IOException e) { LOG.warn("Exception while writing response for params: " + params, e); } finally { - closeNoExp(inputStream); + IOUtils.closeQuietly(inputStream); } } diff --git a/src/java/org/apache/solr/handler/SnapPuller.java b/src/java/org/apache/solr/handler/SnapPuller.java index 4f2fd8a6d67..e587df7d234 100644 --- a/src/java/org/apache/solr/handler/SnapPuller.java +++ b/src/java/org/apache/solr/handler/SnapPuller.java @@ -18,6 +18,7 @@ package org.apache.solr.handler; import org.apache.commons.httpclient.*; import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.io.IOUtils; import org.apache.lucene.index.IndexCommit; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.FastInputStream; @@ -380,8 +381,8 @@ public class SnapPuller { LOG.warn("Exception while updating statistics", e); } finally { - closeNoExp(inFile); - closeNoExp(outFile); + IOUtils.closeQuietly(inFile); + IOUtils.closeQuietly(outFile); } } @@ -584,7 +585,7 @@ public class SnapPuller { } catch (Exception e) { LOG.error("Unable to load index.properties"); } finally { - closeNoExp(is); + IOUtils.closeQuietly(is); } } p.put("index", snap); @@ -596,7 +597,7 @@ public class SnapPuller { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to write index.properties", e); } finally { - closeNoExp(os); + IOUtils.closeQuietly(os); } } @@ -818,7 +819,7 @@ public class SnapPuller { } //if there is an error continue. But continue from the point where it got broken } finally { - closeNoExp(is); + IOUtils.closeQuietly(is); } } } finally { diff --git a/src/java/org/apache/solr/handler/SnapShooter.java b/src/java/org/apache/solr/handler/SnapShooter.java index 4d3a439f8ab..37f7e731ec2 100644 --- a/src/java/org/apache/solr/handler/SnapShooter.java +++ b/src/java/org/apache/solr/handler/SnapShooter.java @@ -16,86 +16,85 @@ */ package org.apache.solr.handler; -import org.apache.solr.common.SolrException; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.solr.core.SolrCore; +import org.apache.solr.common.util.NamedList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Date; -import java.util.WeakHashMap; +import java.util.ArrayList; /** - *

- * Provides functionality equivalent to the snapshooter script - *

+ *

Provides functionality equivalent to the snapshooter script

* * @version $Id$ * @since solr 1.4 */ public class SnapShooter { + private static final Logger LOG = LoggerFactory.getLogger(SnapShooter.class.getName()); private String snapDir = null; private SolrCore solrCore; + private SimpleFSLockFactory lockFactory; - public SnapShooter(SolrCore core) { + public SnapShooter(SolrCore core) throws IOException { solrCore = core; snapDir = core.getDataDir(); + lockFactory = new SimpleFSLockFactory(snapDir); } - void createSnapAsync(final Collection files) { + void createSnapAsync(final Collection files, final ReplicationHandler replicationHandler) { new Thread() { public void run() { - createSnapshot(files); + createSnapshot(files, replicationHandler); } }.start(); } - void createSnapshot(Collection files) { - File lockFile = null; + void createSnapshot(Collection files, ReplicationHandler replicationHandler) { + NamedList details = new NamedList(); + details.add("startTime", new Date().toString()); File snapShotDir = null; String directoryName = null; + Lock lock = null; try { - lockFile = new File(snapDir, directoryName + ".lock"); - if (lockFile.exists()) { - return; - } SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT); directoryName = "snapshot." + fmt.format(new Date()); + lock = lockFactory.makeLock(directoryName + ".lock"); + if (lock.isLocked()) return; snapShotDir = new File(snapDir, directoryName); - lockFile.createNewFile(); - snapShotDir.mkdir(); - for (String indexFile : files) { - copyFile2Dir(new File(solrCore.getIndexDir(), indexFile), snapShotDir); + if (!snapShotDir.mkdir()) { + LOG.warn("Unable to create snapshot directory: " + snapShotDir.getAbsolutePath()); + return; } + for (String indexFile : files) { + FileUtils.copyFileToDirectory(new File(solrCore.getIndexDir(), indexFile), snapShotDir, true); + } + details.add("fileCount", files.size()); + details.add("status", "success"); + details.add("snapshotCompletedAt", new Date().toString()); } catch (Exception e) { SnapPuller.delTree(snapShotDir); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + LOG.error("Exception while creating snapshot", e); + details.add("snapShootException", e.getMessage()); } finally { - if (lockFile != null) { - lockFile.delete(); + replicationHandler.snapShootDetails = details; + if (lock != null) { + try { + lock.release(); + } catch (IOException e) { + LOG.error("Unable to release snapshoot lock: " + directoryName + ".lock"); + } } } } - static void copyFile2Dir(File file, File toDir) throws IOException { - FileInputStream fis = null; - FileOutputStream fos = null; - try { - fis = new FileInputStream(file); - File destFile = new File(toDir, file.getName()); - fos = new FileOutputStream(destFile); - fis.getChannel().transferTo(0, fis.available(), fos.getChannel()); - destFile.setLastModified(file.lastModified()); - } finally { - ReplicationHandler.closeNoExp(fis); - ReplicationHandler.closeNoExp(fos); - } - } - public static final String SNAP_DIR = "snapDir"; public static final String DATE_FMT = "yyyyMMddhhmmss"; - private static WeakHashMap SNAP_DIRS = new WeakHashMap(); }