Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-12312: Replication's IndexFetcher buf size should be initialized 

to an amount no greater than the size of the file being transferred.
This commit is contained in:
David Smiley 2018-05-07 14:54:11 -04:00
parent b0b32931b2
commit 81f611209c
2 changed files with 14 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -232,6 +232,9 @@ Optimizations
* SOLR-12066: Cleanup deleted core when node start (Cao Manh Dat)
* SOLR-12312: Replication IndexFetcher should cap its internal buffer size when the file being transferred is small.
(Jeff Miller, David Smiley)
Other Changes
----------------------

40
solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
View File

@ -16,31 +16,6 @@
*/
package org.apache.solr.handler;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.handler.ReplicationHandler.ALIAS;
import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE;
import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST;
import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION;
import static org.apache.solr.handler.ReplicationHandler.COMMAND;
import static org.apache.solr.handler.ReplicationHandler.COMPRESSION;
import static org.apache.solr.handler.ReplicationHandler.CONF_FILES;
import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT;
import static org.apache.solr.handler.ReplicationHandler.EXTERNAL;
import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER;
import static org.apache.solr.handler.ReplicationHandler.FILE;
import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
import static org.apache.solr.handler.ReplicationHandler.GENERATION;
import static org.apache.solr.handler.ReplicationHandler.INTERNAL;
import static org.apache.solr.handler.ReplicationHandler.MASTER_URL;
import static org.apache.solr.handler.ReplicationHandler.OFFSET;
import static org.apache.solr.handler.ReplicationHandler.SIZE;
import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO;
import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE;
import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@ -79,6 +54,7 @@ import java.util.zip.Adler32;
import java.util.zip.Checksum;
import java.util.zip.InflaterInputStream;
import com.google.common.base.Strings;
import org.apache.http.client.HttpClient;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexCommit;
@ -111,7 +87,7 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ReplicationHandler.FileInfo;
import org.apache.solr.handler.ReplicationHandler.*;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
@ -128,7 +104,9 @@ import org.apache.solr.util.TestInjection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Strings;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.handler.ReplicationHandler.*;
/**
* <p> Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the
@ -1539,7 +1517,7 @@ public class IndexFetcher {
private final long size;
private long bytesDownloaded = 0;
private byte[] buf = new byte[1024 * 1024];
private byte[] buf;
private final Checksum checksum;
private int errorCount = 0;
private boolean aborted = false;
@ -1549,6 +1527,7 @@ public class IndexFetcher {
this.file = file;
this.fileName = (String) fileDetails.get(NAME);
this.size = (Long) fileDetails.get(SIZE);
buf = new byte[(int)Math.min(this.size, ReplicationHandler.PACKET_SZ)];
this.solrParamOutput = solrParamOutput;
this.saveAs = saveAs;
indexGen = latestGen;
@ -1629,8 +1608,11 @@ public class IndexFetcher {
LOG.warn("No content received for file: {}", fileName);
return NO_CONTENT;
}
if (buf.length < packetSize)
//TODO consider recoding the remaining logic to not use/need buf[]; instead use the internal buffer of fis
if (buf.length < packetSize) {
//This shouldn't happen since sender should use PACKET_SZ and we init the buf based on that too
buf = new byte[packetSize];
}
if (checksum != null) {
//read the checksum
fis.readFully(longbytes);