Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3923: Faster SVN WC checks 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1377991 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2012-08-28 06:50:46 +00:00
parent d3e059b5e7
commit d8f36a22b1
9 changed files with 127 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
build-clover.xml
View File

@ -1,49 +0,0 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project name="clover" basedir=".">
<import file="lucene/common-build.xml"/>
<!--
Run after Junit tests.
This target is in a separate file, as it needs to include common-build.xml,
but must run from top-level!
-->
<target name="generate-clover-reports" depends="clover">
<fail unless="run.clover">Clover not enabled!</fail>
<mkdir dir="${clover.report.dir}"/>
<fileset dir="." id="clover.test.result.files">
<include name="*/build/**/test/TEST-*.xml"/>
<exclude name="lucene/build/backwards/**"/>
</fileset>
<clover-report>
<current outfile="${clover.report.dir}" title="${final.name}" numThreads="0">
<format type="html" filter="assert"/>
<testresults refid="clover.test.result.files"/>
</current>
<current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
<format type="xml" filter="assert"/>
<testresults refid="clover.test.result.files"/>
</current>
</clover-report>
<echo>You can find the merged Lucene/Solr Clover report in '${clover.report.dir}'.</echo>
</target>
</project>

45
build.xml
View File

@ -75,12 +75,6 @@
<fail if="validate.patternsFound">The following files contain @author tags or nocommits:${line.separator}${validate.patternsFound}</fail> <fail if="validate.patternsFound">The following files contain @author tags or nocommits:${line.separator}${validate.patternsFound}</fail>
</target> </target>
<target name="check-svn-properties">
<subant target="-check-svn-properties" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml" />
</subant>
</target>
<target name="rat-sources" description="Runs rat across all sources and tests"> <target name="rat-sources" description="Runs rat across all sources and tests">
<sequential><subant target="rat-sources" inheritall="false" failonerror="true"> <sequential><subant target="rat-sources" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml" /> <fileset dir="lucene" includes="build.xml" />
@ -254,17 +248,21 @@
</sequential> </sequential>
</target> </target>
<target name="check-svn-working-copy">
<subant target="check-svn-working-copy" inheritall="false" failonerror="true">
<fileset dir="." includes="extra-targets.xml" />
</subant>
</target>
<!-- Calls only generate-clover-reports on Lucene, as Solr's is just a clone with other target; the database itsself is fixed --> <!-- Calls only generate-clover-reports on Lucene, as Solr's is just a clone with other target; the database itsself is fixed -->
<target name="generate-clover-reports"> <target name="generate-clover-reports">
<subant target="generate-clover-reports" inheritall="false" failonerror="true"> <subant target="generate-clover-reports" inheritall="false" failonerror="true">
<fileset dir="." includes="build-clover.xml" /> <fileset dir="." includes="extra-targets.xml" />
</subant> </subant>
</target> </target>
<!-- Jenkins tasks --> <!-- Jenkins tasks -->
<!-- TODO: figure out how to run check-svn-properties for the various jenkins' that <target name="jenkins-hourly" depends="clean,test,validate,-jenkins-javadocs-lint,check-svn-working-copy"/>
are configured in slow ways / have slow i/o systems -->
<target name="jenkins-hourly" depends="clean,test,validate,-jenkins-javadocs-lint,-svn-status"/>
<target name="jenkins-clover"> <target name="jenkins-clover">
<antcall target="-jenkins-clover"> <antcall target="-jenkins-clover">
@ -288,31 +286,4 @@
<target name="-jenkins-javadocs-lint" unless="-disable.javadocs-lint"> <target name="-jenkins-javadocs-lint" unless="-disable.javadocs-lint">
<antcall target="javadocs-lint"/> <antcall target="javadocs-lint"/>
</target> </target>
<!-- define here, as common-build is not included! -->
<property name="svn.exe" value="svn" />
<target name="-svn-status">
<exec executable="${svn.exe}" dir="." failonerror="true">
<arg value="status"/>
<redirector outputproperty="svn.status.output">
<outputfilterchain>
<linecontainsregexp>
<regexp pattern="^\?" />
</linecontainsregexp>
<tokenfilter>
<replaceregex pattern="^........" replace="* " />
<replacestring from="${file.separator}" to="/" />
</tokenfilter>
</outputfilterchain>
</redirector>
</exec>
<fail message="Source checkout is dirty after running tests!!! Offending files:${line.separator}${svn.status.output}">
<condition>
<not>
<equals arg1="${svn.status.output}" arg2=""/>
</not>
</condition>
</fail>
</target>
</project> </project>

114
extra-targets.xml Normal file
View File

@ -0,0 +1,114 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project name="extra-targets" basedir=".">
<description>
This file is designed for importing into a main build file, and not intended
for standalone use.
</description>
<import file="lucene/common-build.xml"/>
<!--
Run after Junit tests.
This target is in a separate file, as it needs to include common-build.xml,
but must run from top-level!
-->
<target name="generate-clover-reports" depends="clover">
<fail unless="run.clover">Clover not enabled!</fail>
<mkdir dir="${clover.report.dir}"/>
<fileset dir="." id="clover.test.result.files">
<include name="*/build/**/test/TEST-*.xml"/>
<exclude name="lucene/build/backwards/**"/>
</fileset>
<clover-report>
<current outfile="${clover.report.dir}" title="${final.name}" numThreads="0">
<format type="html" filter="assert"/>
<testresults refid="clover.test.result.files"/>
</current>
<current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
<format type="xml" filter="assert"/>
<testresults refid="clover.test.result.files"/>
</current>
</clover-report>
<echo>You can find the merged Lucene/Solr Clover report in '${clover.report.dir}'.</echo>
</target>
<target xmlns:ivy="antlib:org.apache.ivy.ant" name="check-svn-working-copy" depends="ivy-availability-check,ivy-fail,ivy-configure">
<ivy:cachepath organisation="org.tmatesoft.svnkit" module="svnkit" revision="1.7.5-v1"
inline="true" conf="default" type="jar" transitive="true" pathid="svnkit.classpath"/>
<script language="javascript" classpathref="svnkit.classpath" taskname="svn"><![CDATA[
importClass(java.io.File);
importClass(java.util.TreeSet);
importPackage(org.tmatesoft.svn.core);
importPackage(org.tmatesoft.svn.core.wc);
var manager = SVNClientManager.newInstance();
var statusClient = manager.getStatusClient();
var wcClient = manager.getWCClient();
var basedir = new File(project.getProperty("basedir")).getAbsoluteFile();
var baseLen = basedir.toString().length();
var convertRelative = function(file) {
return file.getAbsolutePath().substring(baseLen + 1).replace(File.separatorChar, '/');
}
var missingProps = new TreeSet(), unversioned = new TreeSet();
self.log("Getting all versioned and unversioned files...");
statusClient.doStatus(basedir, SVNRevision.WORKING, SVNDepth.fromRecurse(true), false, true, false, false, new ISVNStatusHandler({
handleStatus: function(status) {
var nodeStatus = status.getNodeStatus();
if (nodeStatus == SVNStatusType.STATUS_UNVERSIONED) {
unversioned.add(convertRelative(status.getFile()));
} else if (status.getKind() == SVNNodeKind.FILE && nodeStatus != SVNStatusType.STATUS_DELETED) {
missingProps.add(convertRelative(status.getFile()));
}
}
}), null);
self.log("Filtering files with existing svn:eol-style...");
wcClient.doGetProperty(basedir, "svn:eol-style", SVNRevision.WORKING, SVNRevision.WORKING, true, new ISVNPropertyHandler({
handleProperty: function(file, prop) {
missingProps.remove(convertRelative(file));
}
}));
self.log("Filtering files with binary svn:mime-type...");
wcClient.doGetProperty(basedir, "svn:mime-type", SVNRevision.WORKING, SVNRevision.WORKING, true, new ISVNPropertyHandler({
handleProperty: function(file, prop) {
prop = SVNPropertyValue.getPropertyAsString(prop.getValue());
if (prop.startsWith("application/") || prop.startsWith("image/")) {
missingProps.remove(convertRelative(file));
}
}
}));
var convertSet2String = function(set) {
return set.isEmpty() ? null : ("* " + set.toArray().join(project.getProperty("line.separator") + "* "))
};
project.setProperty("svn.checkprops.failed", convertSet2String(missingProps));
project.setProperty("svn.unversioned.failed", convertSet2String(unversioned));
]]></script>
<fail if="svn.checkprops.failed"
message="The following files are missing svn:eol-style (or binary svn:mime-type):${line.separator}${svn.checkprops.failed}"/>
<fail if="svn.unversioned.failed"
message="Source checkout is dirty after running tests!!! Offending files:${line.separator}${svn.unversioned.failed}"/>
</target>
</project>

10
lucene/build.xml
View File

@ -198,16 +198,6 @@
</forbidden-apis> </forbidden-apis>
</target> </target>
<!-- note: we don't include this in validate because we want to check from releases -->
<target name="-check-svn-properties" depends="compile-tools,resolve,load-custom-tasks">
<svn-eol-style svnExecutable="${svn.exe}">
<fileset dir="${basedir}/..">
<exclude name="**/build/**"/>
<exclude name="**/*.jar"/>
</fileset>
</svn-eol-style>
</target>
<target name="resolve"> <target name="resolve">
<sequential> <sequential>
<ant dir="test-framework" target="resolve" inheritall="false"> <ant dir="test-framework" target="resolve" inheritall="false">

13
lucene/common-build.xml
View File

@ -508,6 +508,9 @@
<attribute name="spec.version"/> <attribute name="spec.version"/>
<attribute name="manifest.file" default="${manifest.file}"/> <attribute name="manifest.file" default="${manifest.file}"/>
<sequential> <sequential>
<!-- If possible, include the svnversion -->
<exec dir="." executable="${svnversion.exe}" outputproperty="svnversion" failifexecutionfails="false"/>
<manifest file="@{manifest.file}"> <manifest file="@{manifest.file}">
<!-- <!--
http://java.sun.com/j2se/1.5.0/docs/guide/jar/jar.html#JAR%20Manifest http://java.sun.com/j2se/1.5.0/docs/guide/jar/jar.html#JAR%20Manifest
@ -558,12 +561,6 @@
<attribute name="manifest.file" default="${manifest.file}"/> <attribute name="manifest.file" default="${manifest.file}"/>
<element name="nested" optional="true" implicit="true"/> <element name="nested" optional="true" implicit="true"/>
<sequential> <sequential>
<!-- If possible, include the svnversion -->
<exec dir="." executable="${svnversion.exe}"
outputproperty="svnversion" failifexecutionfails="false">
<arg value="."/>
</exec>
<build-manifest title="@{title}" <build-manifest title="@{title}"
implementation.title="@{implementation.title}" implementation.title="@{implementation.title}"
spec.version="@{spec.version}" spec.version="@{spec.version}"
@ -1547,10 +1544,10 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites
description="Populates properties svn.URL and svn.Revision using 'svn info'."> description="Populates properties svn.URL and svn.Revision using 'svn info'.">
<attribute name="directory"/> <attribute name="directory"/>
<sequential> <sequential>
<exec dir="." executable="${svnversion.exe}" outputproperty="svn.ver"/> <exec dir="@{directory}" executable="${svnversion.exe}" outputproperty="svn.ver"/>
<fail message="A subversion checkout is required for this target"> <fail message="A subversion checkout is required for this target">
<condition> <condition>
<equals arg1="${svn.ver}" arg2="exported"/> <matches pattern="(exported|unversioned.*)" string="${svn.ver}" casesensitive="false"/>
</condition> </condition>
</fail> </fail>
<exec dir="@{directory}" executable="${svn.exe}" outputproperty="svn.info" failonerror="true"> <exec dir="@{directory}" executable="${svn.exe}" outputproperty="svn.info" failonerror="true">

3
lucene/tools/src/java/lucene-solr.antlib.xml
View File

@ -21,7 +21,4 @@
<taskdef <taskdef
name="forbidden-apis" name="forbidden-apis"
classname="org.apache.lucene.validation.ForbiddenApisCheckTask" /> classname="org.apache.lucene.validation.ForbiddenApisCheckTask" />
<taskdef
name="svn-eol-style"
classname="org.apache.lucene.validation.SVNEolCheckTask" />
</antlib> </antlib>

126
lucene/tools/src/java/org/apache/lucene/validation/SVNEolCheckTask.java
View File

@ -1,126 +0,0 @@
package org.apache.lucene.validation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.types.Resource;
import org.apache.tools.ant.types.ResourceCollection;
import org.apache.tools.ant.types.resources.FileResource;
import org.apache.tools.ant.types.resources.Resources;
/**
* Checks all files to ensure they have svn:eol-style, or
* have a binary svn:mime-type.
* <p>
* TODO: check that this value is actually correct, not just present.
* <p>
* WARNING: slow!
*/
public class SVNEolCheckTask extends Task {
private final Resources files = new Resources();
private String svnExecutable;
/** Set of files to check */
public void add(ResourceCollection rc) {
files.add(rc);
}
/** svn.exe executable */
public void setSvnExecutable(String svnExecutable) {
this.svnExecutable = svnExecutable;
}
@Override
public void execute() throws BuildException {
if (svnExecutable == null) {
throw new BuildException("svnExecutable parameter must be set!");
}
boolean success = true;
files.setProject(getProject());
Iterator<Resource> iter = (Iterator<Resource>) files.iterator();
while (iter.hasNext()) {
Resource r = iter.next();
if (!(r instanceof FileResource)) {
throw new BuildException("Only filesystem resource are supported: " + r.getName()
+ ", was: " + r.getClass().getName());
}
File f = ((FileResource) r).getFile();
List<String> cmd = new ArrayList<String>();
cmd.add(svnExecutable);
cmd.add("pget");
cmd.add("svn:eol-style");
cmd.add(f.getAbsolutePath());
String eolStyle = exec(cmd);
if (eolStyle.isEmpty()) {
cmd.clear();
cmd.add(svnExecutable);
cmd.add("pget");
cmd.add("svn:mime-type");
cmd.add(f.getAbsolutePath());
String binProp = exec(cmd);
if (!binProp.startsWith("application/") && !binProp.startsWith("image/")) {
success = false;
log(r.getName() + " missing svn:eol-style (or binary svn:mime-type).");
}
}
}
if (!success) {
throw new BuildException("Some svn properties are missing");
}
}
private String exec(List<String> cmd) throws BuildException {
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.redirectErrorStream(true);
BufferedReader r = null;
StringBuilder sb = new StringBuilder();
try {
Process p = pb.start();
InputStream is = p.getInputStream();
r = new BufferedReader(new InputStreamReader(is, Charset.defaultCharset()));
int ch;
while ((ch = r.read()) > 0) {
sb.append((char)ch);
}
p.waitFor();
return sb.toString();
} catch (Exception e) {
throw new BuildException(e);
} finally {
if (r != null) {
try {
r.close();
} catch (IOException e) {}
}
}
}
}

6
solr/build.xml
View File

@ -638,12 +638,6 @@
</exec> </exec>
</target> </target>
<target name="svn-up">
<exec executable="${svn.exe}">
<arg value="update"/>
</exec>
</target>
<property name="analysis-common.res.dir" value="../lucene/analysis/common/src/resources/org/apache/lucene/analysis"/> <property name="analysis-common.res.dir" value="../lucene/analysis/common/src/resources/org/apache/lucene/analysis"/>
<property name="analysis-kuromoji.res.dir" value="../lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis"/> <property name="analysis-kuromoji.res.dir" value="../lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis"/>
<property name="analysis.conf.dest" value="${example}/solr/conf/lang"/> <property name="analysis.conf.dest" value="${example}/solr/conf/lang"/>

4
solr/webapp/build.xml
View File

@ -40,10 +40,6 @@
<target name="dist" <target name="dist"
description="Creates the Solr WAR Distribution file." description="Creates the Solr WAR Distribution file."
depends="test, init-dist, dist-core, dist-solrj, lucene-jars-to-solr"> depends="test, init-dist, dist-core, dist-solrj, lucene-jars-to-solr">
<exec dir="." executable="${svnversion.exe}"
outputproperty="svnversion" failifexecutionfails="false">
<arg line="."/>
</exec>
<build-manifest title="Apache Solr Search Server" <build-manifest title="Apache Solr Search Server"
implementation.title="org.apache.solr" implementation.title="org.apache.solr"
spec.version="${solr.spec.version}"/> spec.version="${solr.spec.version}"/>