From 81f611209c9b71b7538a46d047631ea164dc2a2c Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 7 May 2018 14:54:11 -0400 Subject: [PATCH] SOLR-12312: Replication's IndexFetcher buf size should be initialized to an amount no greater than the size of the file being transferred. --- solr/CHANGES.txt | 3 ++ .../org/apache/solr/handler/IndexFetcher.java | 40 +++++-------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 48f9ad5d910..abc6d6ca12a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -232,6 +232,9 @@ Optimizations * SOLR-12066: Cleanup deleted core when node start (Cao Manh Dat) +* SOLR-12312: Replication IndexFetcher should cap its internal buffer size when the file being transferred is small. + (Jeff Miller, David Smiley) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 93f0edf776b..d26a0dbbb88 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -16,31 +16,6 @@ */ package org.apache.solr.handler; -import static org.apache.solr.common.params.CommonParams.JAVABIN; -import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.ReplicationHandler.ALIAS; -import static org.apache.solr.handler.ReplicationHandler.CHECKSUM; -import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS; -import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE; -import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST; -import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION; -import static org.apache.solr.handler.ReplicationHandler.COMMAND; -import static org.apache.solr.handler.ReplicationHandler.COMPRESSION; -import static org.apache.solr.handler.ReplicationHandler.CONF_FILES; -import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT; -import static org.apache.solr.handler.ReplicationHandler.EXTERNAL; -import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER; -import static org.apache.solr.handler.ReplicationHandler.FILE; -import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; -import static org.apache.solr.handler.ReplicationHandler.GENERATION; -import static org.apache.solr.handler.ReplicationHandler.INTERNAL; -import static org.apache.solr.handler.ReplicationHandler.MASTER_URL; -import static org.apache.solr.handler.ReplicationHandler.OFFSET; -import static org.apache.solr.handler.ReplicationHandler.SIZE; -import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO; -import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE; -import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES; - import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -79,6 +54,7 @@ import java.util.zip.Adler32; import java.util.zip.Checksum; import java.util.zip.InflaterInputStream; +import com.google.common.base.Strings; import org.apache.http.client.HttpClient; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexCommit; @@ -111,7 +87,7 @@ import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.SolrCore; -import org.apache.solr.handler.ReplicationHandler.FileInfo; +import org.apache.solr.handler.ReplicationHandler.*; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.SolrIndexSearcher; @@ -128,7 +104,9 @@ import org.apache.solr.util.TestInjection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Strings; +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.handler.ReplicationHandler.*; /** *

Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the @@ -1539,7 +1517,7 @@ public class IndexFetcher { private final long size; private long bytesDownloaded = 0; - private byte[] buf = new byte[1024 * 1024]; + private byte[] buf; private final Checksum checksum; private int errorCount = 0; private boolean aborted = false; @@ -1549,6 +1527,7 @@ public class IndexFetcher { this.file = file; this.fileName = (String) fileDetails.get(NAME); this.size = (Long) fileDetails.get(SIZE); + buf = new byte[(int)Math.min(this.size, ReplicationHandler.PACKET_SZ)]; this.solrParamOutput = solrParamOutput; this.saveAs = saveAs; indexGen = latestGen; @@ -1629,8 +1608,11 @@ public class IndexFetcher { LOG.warn("No content received for file: {}", fileName); return NO_CONTENT; } - if (buf.length < packetSize) + //TODO consider recoding the remaining logic to not use/need buf[]; instead use the internal buffer of fis + if (buf.length < packetSize) { + //This shouldn't happen since sender should use PACKET_SZ and we init the buf based on that too buf = new byte[packetSize]; + } if (checksum != null) { //read the checksum fis.readFully(longbytes);