Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6393: TransactionLog replay performance on HDFS is very poor. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1619200 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2014-08-20 18:46:27 +00:00
parent cac2292ce0
commit cdd0fe7f8d
2 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -296,6 +296,8 @@ Bug Fixes
* SOLR-6378: Fixed example/example-DIH/ issues with "tika" and "solr" configurations, and tidied up README.txt
(Daniel Shchyokin via ehatcher)
* SOLR-6393: TransactionLog replay performance on HDFS is very poor. (Mark Miller)
Optimizations
---------------------

16
solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java
View File

@ -341,12 +341,14 @@ public class HdfsTransactionLog extends TransactionLog {
public class HDFSLogReader extends LogReader{
FSDataFastInputStream fis;
private LogCodec codec = new LogCodec(resolver);
private long sz;
public HDFSLogReader(long startingPos) {
super();
incref();
try {
FSDataInputStream fdis = fs.open(tlogFile);
sz = fs.getFileStatus(tlogFile).getLen();
fis = new FSDataFastInputStream(fdis, startingPos);
} catch (IOException e) {
throw new RuntimeException(e);
@ -361,7 +363,6 @@ public class HdfsTransactionLog extends TransactionLog {
public Object next() throws IOException, InterruptedException {
long pos = fis.position();
synchronized (HdfsTransactionLog.this) {
if (trace) {
log.trace("Reading log record. pos="+pos+" currentSize="+fos.size());
@ -372,18 +373,22 @@ public class HdfsTransactionLog extends TransactionLog {
}
fos.flushBuffer();
tlogOutStream.hflush();
// we actually need a new reader
}
// we actually need a new reader to
// see if any data was added by the writer
if (fis.position() >= sz) {
fis.close();
tlogOutStream.hflush();
try {
FSDataInputStream fdis = fs.open(tlogFile);
fis = new FSDataFastInputStream(fdis, pos);
sz = fs.getFileStatus(tlogFile).getLen();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (pos == 0) {
readHeader(fis);
@ -396,7 +401,6 @@ public class HdfsTransactionLog extends TransactionLog {
}
}
tlogOutStream.hflush();
Object o = codec.readVal(fis);
// skip over record size