Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1115: some small fixes to contrib/benchmark 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@608306 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2008-01-03 01:48:18 +00:00
parent ed893f770c
commit 263244312d
5 changed files with 89 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocMaker.java
View File

@ -85,10 +85,18 @@ public class LineDocMaker extends BasicDocMaker {
public Document setFields(String line) {
// title <TAB> date <TAB> body <NEWLINE>
int spot = line.indexOf(SEP);
titleField.setValue(line.substring(0, spot));
int spot2 = line.indexOf(SEP, 1+spot);
dateField.setValue(line.substring(1+spot, spot2));
bodyField.setValue(line.substring(1+spot2, line.length()));
if (spot != -1) {
titleField.setValue(line.substring(0, spot));
int spot2 = line.indexOf(SEP, 1+spot);
if (spot2 != -1) {
dateField.setValue(line.substring(1+spot, spot2));
bodyField.setValue(line.substring(1+spot2, line.length()));
} else {
dateField.setValue("");
bodyField.setValue("");
}
} else
titleField.setValue("");
return doc;
}
}
@ -121,12 +129,10 @@ public class LineDocMaker extends BasicDocMaker {
while(true) {
line = fileIn.readLine();
if (line == null) {
// Reset the file
openFile();
if (!forever)
throw new NoMoreDataException();
else {
// Reset the file
openFile();
}
} else {
break;
}

8
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
View File

@ -57,12 +57,8 @@ public class CreateIndexTask extends PerfTask {
iw.setUseCompoundFile(cmpnd);
iw.setMergeFactor(mrgf);
iw.setMaxFieldLength(mxfl);
if (flushAtRAMUsage > 0)
iw.setRAMBufferSizeMB(flushAtRAMUsage);
else if (mxbf != 0)
iw.setMaxBufferedDocs(mxbf);
else
throw new RuntimeException("either max.buffered or ram.flush.mb must be non-zero");
iw.setRAMBufferSizeMB(flushAtRAMUsage);
iw.setMaxBufferedDocs(mxbf);
getRunData().setIndexWriter(iw);
return 1;
}

8
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java
View File

@ -61,12 +61,8 @@ public class OpenIndexTask extends PerfTask {
IndexWriter writer = new IndexWriter(dir, autoCommit, analyzer, false);
// must update params for newly opened writer
if (flushAtRAMUsage > 0)
writer.setRAMBufferSizeMB(flushAtRAMUsage);
else if (mxbf != 0)
writer.setMaxBufferedDocs(mxbf);
else
throw new RuntimeException("either max.buffered or ram.flush.mb must be non-zero");
writer.setRAMBufferSizeMB(flushAtRAMUsage);
writer.setMaxBufferedDocs(mxbf);
writer.setMaxFieldLength(mxfl);
writer.setMergeFactor(mrgf);
writer.setUseCompoundFile(cmpnd); // this one redundant?

61
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
View File

@ -38,6 +38,8 @@ public class TaskSequence extends PerfTask {
private String seqName;
private boolean exhausted = false;
private boolean resetExhausted = false;
private PerfTask[] tasksArray;
private boolean anyExhaustableTasks;
public TaskSequence (PerfRunData runData, String name, TaskSequence parent, boolean parallel) {
super(runData);
@ -49,6 +51,18 @@ public class TaskSequence extends PerfTask {
tasks = new ArrayList();
}
private void initTasksArray() {
if (tasksArray == null) {
final int numTasks = tasks.size();
tasksArray = new PerfTask[numTasks];
for(int k=0;k<numTasks;k++) {
tasksArray[k] = (PerfTask) tasks.get(k);
anyExhaustableTasks |= tasksArray[k] instanceof ResetInputsTask;
anyExhaustableTasks |= tasksArray[k] instanceof TaskSequence;
}
}
}
/**
* @return Returns the parallel.
*/
@ -92,7 +106,7 @@ public class TaskSequence extends PerfTask {
* @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#doLogic()
*/
public int doLogic() throws Exception {
resetExhausted = false;
exhausted = resetExhausted = false;
return ( parallel ? doParallelTasks() : doSerialTasks());
}
@ -101,18 +115,16 @@ public class TaskSequence extends PerfTask {
return doSerialTasksWithRate();
}
initTasksArray();
int count = 0;
final int numTasks = tasks.size();
final PerfTask[] tasksArray = new PerfTask[numTasks];
for(int k=0;k<numTasks;k++)
tasksArray[k] = (PerfTask) tasks.get(k);
for (int k=0; (repetitions==REPEAT_EXHAUST && !exhausted) || k<repetitions; k++) {
for(int l=0;l<numTasks;l++)
for(int l=0;l<tasksArray.length;l++)
try {
count += tasksArray[l].runAndMaybeStats(letChildReport);
updateExhausted(tasksArray[l]);
final PerfTask task = tasksArray[l];
count += task.runAndMaybeStats(letChildReport);
if (anyExhaustableTasks)
updateExhausted(task);
} catch (NoMoreDataException e) {
exhausted = true;
}
@ -121,12 +133,13 @@ public class TaskSequence extends PerfTask {
}
private int doSerialTasksWithRate() throws Exception {
initTasksArray();
long delayStep = (perMin ? 60000 : 1000) /rate;
long nextStartTime = System.currentTimeMillis();
int count = 0;
for (int k=0; (repetitions==REPEAT_EXHAUST && !exhausted) || k<repetitions; k++) {
for (Iterator it = tasks.iterator(); it.hasNext();) {
PerfTask task = (PerfTask) it.next();
for (int l=0;l<tasksArray.length;l++) {
final PerfTask task = tasksArray[l];
long waitMore = nextStartTime - System.currentTimeMillis();
if (waitMore > 0) {
//System.out.println("wait: "+waitMore+" for rate: "+ratePerMin+" (delayStep="+delayStep+")");
@ -135,7 +148,8 @@ public class TaskSequence extends PerfTask {
nextStartTime += delayStep; // this aims at avarage rate.
try {
count += task.runAndMaybeStats(letChildReport);
updateExhausted(task);
if (anyExhaustableTasks)
updateExhausted(task);
} catch (NoMoreDataException e) {
exhausted = true;
}
@ -149,28 +163,27 @@ public class TaskSequence extends PerfTask {
if (task instanceof ResetInputsTask) {
exhausted = false;
resetExhausted = true;
} else {
if (task instanceof TaskSequence) {
TaskSequence t = (TaskSequence) task;
if (t.resetExhausted) {
exhausted = false;
resetExhausted = true;
t.resetExhausted = false;
} else {
exhausted |= t.exhausted;
}
} else if (task instanceof TaskSequence) {
TaskSequence t = (TaskSequence) task;
if (t.resetExhausted) {
exhausted = false;
resetExhausted = true;
t.resetExhausted = false;
} else {
exhausted |= t.exhausted;
}
}
}
private int doParallelTasks() throws Exception {
initTasksArray();
final int count [] = {0};
Thread t[] = new Thread [repetitions * tasks.size()];
// prepare threads
int indx = 0;
for (int k=0; k<repetitions; k++) {
for (int i = 0; i < tasks.size(); i++) {
final PerfTask task = (PerfTask) ((PerfTask) tasks.get(i)).clone();
for (int i = 0; i < tasksArray.length; i++) {
final PerfTask task = (PerfTask) tasksArray[i].clone();
t[indx++] = new Thread() {
public void run() {
int n;

34
contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
View File

@ -359,4 +359,38 @@ public class TestPerfTasksLogic extends TestCase {
nDocs = 0;
}
}
/**
* Test that exhaust in loop works as expected (LUCENE-1115).
*/
public void testExhaustedLooped() throws Exception {
// 1. alg definition (required in every "logic" test)
String algLines[] = {
"# ----- properties ",
"doc.maker="+Reuters20DocMaker.class.getName(),
"doc.add.log.step=3",
"doc.term.vector=false",
"doc.maker.forever=false",
"directory=RAMDirectory",
"doc.stored=false",
"doc.tokenized=false",
"debug.level=1",
"# ----- alg ",
"{ \"Rounds\"",
" ResetSystemErase",
" CreateIndex",
" { \"AddDocs\" AddDoc > : * ",
" CloseIndex",
"} : 2",
};
// 2. execute the algorithm (required in every "logic" test)
Benchmark benchmark = execBenchmark(algLines);
// 3. test number of docs in the index
IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory());
int ndocsExpected = 20; // Reuters20DocMaker exhausts after 20 docs.
assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs());
ir.close();
}
}