Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2419: improve parallel tests 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@939111 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2010-04-28 22:08:59 +00:00
parent c6e0e86acc
commit c6e8519b51
9 changed files with 223 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -497,7 +497,7 @@ Build
can force them to run sequentially by passing -Drunsequential=1 on the command
line. The number of threads that are spwaned per CPU defaults to '1'. If you
wish to change that, you can run the tests with -DthreadsPerProcessor=[num].
(Robert Muir, Shai Erera)
(Robert Muir, Shai Erera, Peter Kofler)
Test Cases

63
lucene/backwards/src/test/org/apache/lucene/util/LuceneJUnitDividingSelector.java Normal file
View File

@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.lucene.util;
import java.io.File;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.types.Parameter;
import org.apache.tools.ant.types.selectors.BaseExtendSelector;
/** Divides filesets into equal groups */
public class LuceneJUnitDividingSelector extends BaseExtendSelector {
private int counter;
/** Number of total parts to split. */
private int divisor;
/** Current part to accept. */
private int part;
public void setParameters(Parameter[] pParameters) {
super.setParameters(pParameters);
for (int j = 0; j < pParameters.length; j++) {
Parameter p = pParameters[j];
if ("divisor".equalsIgnoreCase(p.getName())) {
divisor = Integer.parseInt(p.getValue());
}
else if ("part".equalsIgnoreCase(p.getName())) {
part = Integer.parseInt(p.getValue());
}
else {
throw new BuildException("unknown " + p.getName());
}
}
}
public void verifySettings() {
super.verifySettings();
if (divisor <= 0 || part <= 0) {
throw new BuildException("part or divisor not set");
}
if (part > divisor) {
throw new BuildException("part must be <= divisor");
}
}
public boolean isSelected(File dir, String name, File path) {
counter = counter % divisor + 1;
return counter == part;
}
}

18
lucene/backwards/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java
View File

@ -1,4 +1,4 @@
/*
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@ -43,7 +43,7 @@ import org.apache.tools.ant.util.StringUtils;
public class LuceneJUnitResultFormatter implements JUnitResultFormatter {
private static final double ONE_SECOND = 1000.0;
private NativeFSLockFactory lockFactory;
private static final NativeFSLockFactory lockFactory;
/** Where to write the log to. */
private OutputStream out;
@ -60,13 +60,13 @@ public class LuceneJUnitResultFormatter implements JUnitResultFormatter {
/** Buffer output until the end of the test */
private StringBuilder sb;
private org.apache.lucene.store.Lock lock;
private static final org.apache.lucene.store.Lock lock;
/** Constructor for SolrJUnitResultFormatter. */
public LuceneJUnitResultFormatter() {
File lockDir = new File(System.getProperty("java.io.tmpdir"), "lucene_junit_lock");
static {
File lockDir = new File(System.getProperty("java.io.tmpdir"),
"lucene_junit_lock");
lockDir.mkdirs();
if(!lockDir.exists()) {
if (!lockDir.exists()) {
throw new RuntimeException("Could not make Lock directory:" + lockDir);
}
try {
@ -75,6 +75,10 @@ public class LuceneJUnitResultFormatter implements JUnitResultFormatter {
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** Constructor for LuceneJUnitResultFormatter. */
public LuceneJUnitResultFormatter() {
sb = new StringBuilder();
}

42
lucene/build.xml
View File

@ -131,7 +131,8 @@ The source distribution does not contain sources of the previous Lucene Java ver
</target>
<macrodef name="backwards-test-macro">
<attribute name="pattern" default=""/>
<attribute name="threadNum" default="1"/>
<attribute name="threadTotal" default="1"/>
<sequential>
<!-- run branch tests against trunk jar -->
<test-macro
@ -139,7 +140,8 @@ The source distribution does not contain sources of the previous Lucene Java ver
tempDir="${build.dir.backwards}/test"
junit.classpath="backwards.junit.classpath"
junit.output.dir="${junit.output.dir.backwards}"
pattern="@{pattern}" />
threadNum="@{threadNum}"
threadTotal="@{threadTotal}"/>
</sequential>
</macrodef>
@ -148,33 +150,15 @@ The source distribution does not contain sources of the previous Lucene Java ver
</target>
<target name="junit-backwards-parallel" unless="runsequential">
<parallel threadsPerProcessor="2">
<backwards-test-macro pattern="A"/>
<backwards-test-macro pattern="B"/>
<backwards-test-macro pattern="C"/>
<backwards-test-macro pattern="D"/>
<backwards-test-macro pattern="E"/>
<backwards-test-macro pattern="F"/>
<backwards-test-macro pattern="G"/>
<backwards-test-macro pattern="H"/>
<backwards-test-macro pattern="I"/>
<backwards-test-macro pattern="J"/>
<backwards-test-macro pattern="K"/>
<backwards-test-macro pattern="L"/>
<backwards-test-macro pattern="M"/>
<backwards-test-macro pattern="N"/>
<backwards-test-macro pattern="O"/>
<backwards-test-macro pattern="P"/>
<backwards-test-macro pattern="Q"/>
<backwards-test-macro pattern="R"/>
<backwards-test-macro pattern="S"/>
<backwards-test-macro pattern="T"/>
<backwards-test-macro pattern="U"/>
<backwards-test-macro pattern="V"/>
<backwards-test-macro pattern="W"/>
<backwards-test-macro pattern="X"/>
<backwards-test-macro pattern="Y"/>
<backwards-test-macro pattern="Z"/>
<parallel threadsPerProcessor="${threadsPerProcessor}">
<backwards-test-macro threadNum="1" threadTotal="8"/>
<backwards-test-macro threadNum="2" threadTotal="8"/>
<backwards-test-macro threadNum="3" threadTotal="8"/>
<backwards-test-macro threadNum="4" threadTotal="8"/>
<backwards-test-macro threadNum="5" threadTotal="8"/>
<backwards-test-macro threadNum="6" threadTotal="8"/>
<backwards-test-macro threadNum="7" threadTotal="8"/>
<backwards-test-macro threadNum="8" threadTotal="8"/>
</parallel>
</target>

65
lucene/common-build.xml
View File

@ -117,6 +117,7 @@
<property name="junit.reports.backwards" location="${build.dir.backwards}/test/reports"/>
<property name="junit.excludes" value=""/>
<property name="junit.details.formatter" value="org.apache.lucene.util.LuceneJUnitResultFormatter"/>
<property name="junit.parallel.selector" value="org.apache.lucene.util.LuceneJUnitDividingSelector"/>
<condition property="runsequential">
<or>
<isset property="testcase"/>
@ -410,7 +411,8 @@
<attribute name="junit.classpath" default="junit.classpath"/>
<attribute name="dataDir" default="src/test"/>
<attribute name="tempDir" default="${build.dir}/test"/>
<attribute name="pattern" default=""/>
<attribute name="threadNum" default="1"/>
<attribute name="threadTotal" default="1"/>
<sequential>
<condition property="runall">
@ -423,8 +425,8 @@
<!-- <mkdir dir="@{tempDir}/@{pattern}"/>
This is very loud and obnoxious. abuse touch instead for a "quiet" mkdir
-->
<touch file="@{tempDir}/@{pattern}/quiet.ant" verbose="false" mkdirs="true"/>
<junit printsummary="off" haltonfailure="no" maxmemory="512M" tempdir="@{tempDir}/@{pattern}"
<touch file="@{tempDir}/@{threadNum}/quiet.ant" verbose="false" mkdirs="true"/>
<junit printsummary="off" haltonfailure="no" maxmemory="512M" tempdir="@{tempDir}/@{threadNum}"
errorProperty="tests.failed" failureProperty="tests.failed" forkmode="perBatch" dir=".">
<classpath refid="@{junit.classpath}"/>
<assertions>
@ -437,20 +439,35 @@
<sysproperty key="tests.verbose" value="${tests.verbose}"/>
<!-- TODO: create propertyset for test properties, so each project can have its own set -->
<sysproperty key="tempDir" file="@{tempDir}/@{pattern}"/>
<sysproperty key="tempDir" file="@{tempDir}/@{threadNum}"/>
<sysproperty key="lucene.version" value="${dev.version}"/>
<formatter type="xml"/>
<formatter classname="${junit.details.formatter}" usefile="false"/>
<batchtest fork="yes" todir="@{junit.output.dir}" if="runall">
<fileset dir="@{dataDir}" includes="**/Test@{pattern}*.java,**/@{pattern}*Test.java" excludes="${junit.excludes}"/>
<fileset dir="@{dataDir}" includes="**/Test*.java,**/*Test.java" excludes="${junit.excludes}">
<custom classname="${junit.parallel.selector}" classpathref="@{junit.classpath}">
<param name="divisor" value="@{threadTotal}" />
<param name="part" value="@{threadNum}" />
</custom>
</fileset>
</batchtest>
<batchtest fork="yes" todir="@{junit.output.dir}" if="testpackage">
<fileset dir="@{dataDir}" includes="**/${testpackage}/**/Test@{pattern}*.java,**/${testpackage}/**/@{pattern}*Test.java" excludes="${junit.excludes}"/>
<fileset dir="@{dataDir}" includes="**/${testpackage}/**/Test*.java,**/${testpackage}/**/*Test.java" excludes="${junit.excludes}">
<custom classname="${junit.parallel.selector}" classpathref="@{junit.classpath}">
<param name="divisor" value="@{threadTotal}" />
<param name="part" value="@{threadNum}" />
</custom>
</fileset>
</batchtest>
<batchtest fork="yes" todir="@{junit.output.dir}" if="testpackageroot">
<fileset dir="@{dataDir}" includes="**/${testpackageroot}/Test@{pattern}*.java,**/${testpackageroot}/@{pattern}*Test.java" excludes="${junit.excludes}"/>
<fileset dir="@{dataDir}" includes="**/${testpackageroot}/Test*.java,**/${testpackageroot}/*Test.java" excludes="${junit.excludes}">
<custom classname="${junit.parallel.selector}" classpathref="@{junit.classpath}">
<param name="divisor" value="@{threadTotal}" />
<param name="part" value="@{threadNum}" />
</custom>
</fileset>
</batchtest>
<batchtest fork="yes" todir="@{junit.output.dir}" if="testcase">
<fileset dir="@{dataDir}" includes="**/${testcase}.java"/>
@ -472,32 +489,14 @@
<target name="junit-parallel" unless="runsequential">
<parallel threadsPerProcessor="${threadsPerProcessor}">
<test-macro pattern="A"/>
<test-macro pattern="B"/>
<test-macro pattern="C"/>
<test-macro pattern="D"/>
<test-macro pattern="E"/>
<test-macro pattern="F"/>
<test-macro pattern="G"/>
<test-macro pattern="H"/>
<test-macro pattern="I"/>
<test-macro pattern="J"/>
<test-macro pattern="K"/>
<test-macro pattern="L"/>
<test-macro pattern="M"/>
<test-macro pattern="N"/>
<test-macro pattern="O"/>
<test-macro pattern="P"/>
<test-macro pattern="Q"/>
<test-macro pattern="R"/>
<test-macro pattern="S"/>
<test-macro pattern="T"/>
<test-macro pattern="U"/>
<test-macro pattern="V"/>
<test-macro pattern="W"/>
<test-macro pattern="X"/>
<test-macro pattern="Y"/>
<test-macro pattern="Z"/>
<test-macro threadNum="1" threadTotal="8"/>
<test-macro threadNum="2" threadTotal="8"/>
<test-macro threadNum="3" threadTotal="8"/>
<test-macro threadNum="4" threadTotal="8"/>
<test-macro threadNum="5" threadTotal="8"/>
<test-macro threadNum="6" threadTotal="8"/>
<test-macro threadNum="7" threadTotal="8"/>
<test-macro threadNum="8" threadTotal="8"/>
</parallel>
</target>

63
lucene/src/test/org/apache/lucene/util/LuceneJUnitDividingSelector.java Normal file
View File

@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.lucene.util;
import java.io.File;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.types.Parameter;
import org.apache.tools.ant.types.selectors.BaseExtendSelector;
/** Divides filesets into equal groups */
public class LuceneJUnitDividingSelector extends BaseExtendSelector {
private int counter;
/** Number of total parts to split. */
private int divisor;
/** Current part to accept. */
private int part;
public void setParameters(Parameter[] pParameters) {
super.setParameters(pParameters);
for (int j = 0; j < pParameters.length; j++) {
Parameter p = pParameters[j];
if ("divisor".equalsIgnoreCase(p.getName())) {
divisor = Integer.parseInt(p.getValue());
}
else if ("part".equalsIgnoreCase(p.getName())) {
part = Integer.parseInt(p.getValue());
}
else {
throw new BuildException("unknown " + p.getName());
}
}
}
public void verifySettings() {
super.verifySettings();
if (divisor <= 0 || part <= 0) {
throw new BuildException("part or divisor not set");
}
if (part > divisor) {
throw new BuildException("part must be <= divisor");
}
}
public boolean isSelected(File dir, String name, File path) {
counter = counter % divisor + 1;
return counter == part;
}
}

18
lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java
View File

@ -1,4 +1,4 @@
/*
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@ -43,7 +43,7 @@ import org.apache.tools.ant.util.StringUtils;
public class LuceneJUnitResultFormatter implements JUnitResultFormatter {
private static final double ONE_SECOND = 1000.0;
private NativeFSLockFactory lockFactory;
private static final NativeFSLockFactory lockFactory;
/** Where to write the log to. */
private OutputStream out;
@ -60,13 +60,13 @@ public class LuceneJUnitResultFormatter implements JUnitResultFormatter {
/** Buffer output until the end of the test */
private StringBuilder sb;
private org.apache.lucene.store.Lock lock;
private static final org.apache.lucene.store.Lock lock;
/** Constructor for SolrJUnitResultFormatter. */
public LuceneJUnitResultFormatter() {
File lockDir = new File(System.getProperty("java.io.tmpdir"), "lucene_junit_lock");
static {
File lockDir = new File(System.getProperty("java.io.tmpdir"),
"lucene_junit_lock");
lockDir.mkdirs();
if(!lockDir.exists()) {
if (!lockDir.exists()) {
throw new RuntimeException("Could not make Lock directory:" + lockDir);
}
try {
@ -75,6 +75,10 @@ public class LuceneJUnitResultFormatter implements JUnitResultFormatter {
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** Constructor for LuceneJUnitResultFormatter. */
public LuceneJUnitResultFormatter() {
sb = new StringBuilder();
}

58
solr/build.xml
View File

@ -391,32 +391,14 @@
<target name="junit-parallel" unless="runsequential">
<parallel threadsPerProcessor="${threadsPerProcessor}">
<junit-macro pattern="S"/>
<junit-macro pattern="D"/>
<junit-macro pattern="A"/>
<junit-macro pattern="B"/>
<junit-macro pattern="C"/>
<junit-macro pattern="E"/>
<junit-macro pattern="F"/>
<junit-macro pattern="G"/>
<junit-macro pattern="H"/>
<junit-macro pattern="I"/>
<junit-macro pattern="J"/>
<junit-macro pattern="K"/>
<junit-macro pattern="L"/>
<junit-macro pattern="M"/>
<junit-macro pattern="N"/>
<junit-macro pattern="O"/>
<junit-macro pattern="P"/>
<junit-macro pattern="Q"/>
<junit-macro pattern="R"/>
<junit-macro pattern="T"/>
<junit-macro pattern="U"/>
<junit-macro pattern="V"/>
<junit-macro pattern="W"/>
<junit-macro pattern="X"/>
<junit-macro pattern="Y"/>
<junit-macro pattern="Z"/>
<junit-macro threadNum="1" threadTotal="8"/>
<junit-macro threadNum="2" threadTotal="8"/>
<junit-macro threadNum="3" threadTotal="8"/>
<junit-macro threadNum="4" threadTotal="8"/>
<junit-macro threadNum="5" threadTotal="8"/>
<junit-macro threadNum="6" threadTotal="8"/>
<junit-macro threadNum="7" threadTotal="8"/>
<junit-macro threadNum="8" threadTotal="8"/>
</parallel>
</target>
@ -425,7 +407,8 @@
</target>
<macrodef name="junit-macro">
<attribute name="pattern" default=""/>
<attribute name="threadNum" default="1"/>
<attribute name="threadTotal" default="1"/>
<sequential>
<!-- no description so it doesn't show up in -projecthelp -->
<condition property="runall">
@ -462,13 +445,28 @@
</assertions>
<formatter type="${junit.formatter}"/>
<batchtest fork="yes" todir="${junit.output.dir}" if="runall">
<fileset dir="src/test" includes="**/Test@{pattern}*.java,**/@{pattern}*Test.java"/>
<fileset dir="src/test" includes="**/Test*.java,**/*Test.java">
<custom classname="${junit.parallel.selector}" classpathref="test.run.classpath">
<param name="divisor" value="@{threadTotal}" />
<param name="part" value="@{threadNum}" />
</custom>
</fileset>
</batchtest>
<batchtest fork="yes" todir="${junit.output.dir}" if="testpackage">
<fileset dir="src/test" includes="**/${testpackage}/**/Test@{pattern}*.java,**/${testpackage}/**/@{pattern}*Test.java"/>
<fileset dir="src/test" includes="**/${testpackage}/**/Test*.java,**/${testpackage}/**/*Test.java">
<custom classname="${junit.parallel.selector}" classpathref="test.run.classpath">
<param name="divisor" value="@{threadTotal}" />
<param name="part" value="@{threadNum}" />
</custom>
</fileset>
</batchtest>
<batchtest fork="yes" todir="${junit.output.dir}" if="testpackageroot">
<fileset dir="src/test" includes="**/${testpackageroot}/Test@{pattern}*.java,**/${testpackageroot}/@{pattern}*Test.java"/>
<fileset dir="src/test" includes="**/${testpackageroot}/Test*.java,**/${testpackageroot}/*Test.java">
<custom classname="${junit.parallel.selector}" classpathref="test.run.classpath">
<param name="divisor" value="@{threadTotal}" />
<param name="part" value="@{threadNum}" />
</custom>
</fileset>
</batchtest>
<batchtest fork="yes" todir="${junit.output.dir}" if="testcase">
<fileset dir="src/test" includes="**/${testcase}.java"/>

1
solr/common-build.xml
View File

@ -107,6 +107,7 @@
<property name="junit.reports" location="${common-solr.dir}/${dest}/test-results/reports"/>
<property name="junit.formatter" value="plain"/>
<property name="junit.details.formatter" value="org.apache.lucene.util.LuceneJUnitResultFormatter"/>
<property name="junit.parallel.selector" value="org.apache.lucene.util.LuceneJUnitDividingSelector"/>
<!-- Maven properties -->
<property name="maven.build.dir" value="${basedir}/build/maven"/>