Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4199: commit uwe's patch with check-forbidden-apis task 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4199@1358549 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-07-07 12:04:22 +00:00
parent 752b684960
commit f91f2e9cdb
12 changed files with 429 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lucene/build.xml
View File

@ -169,11 +169,19 @@
</clover-report>
</target>
<!-- Validate once from top-level. -->
<target name="validate" depends="compile-tools,resolve" description="Validate legal stuff.">
<!-- Validation (license/notice/api checks). -->
<target name="validate" depends="check-licenses,check-forbidden-apis" description="Validate stuff." />
<target name="check-licenses" depends="compile-tools,resolve,load-custom-tasks" description="Validate license stuff.">
<license-check-macro dir="${basedir}" />
</target>
<target name="check-forbidden-apis" depends="compile-tools,compile-test,load-custom-tasks" description="Check forbidden API calls in compiled class files.">
<forbidden-apis apiFile="${custom-tasks.dir}/forbiddenApis/jdk.txt">
<fileset dir="${basedir}" includes="**/build/**/*.class" />
</forbidden-apis>
</target>
<target name="resolve">
<sequential>
<ant dir="test-framework" target="resolve" inheritall="false">

11
lucene/tools/build.xml
View File

@ -24,7 +24,11 @@
<import file="../common-build.xml"/>
<path id="classpath"/>
<path id="classpath">
<fileset dir="lib">
<include name="asm-debug-all-4.0.jar"/>
</fileset>
</path>
<!--
Specialize compile-core to not depend on clover, to exclude a
@ -32,8 +36,9 @@
non-existent resource files to the build output directory.
-->
<target name="compile-core" depends="init" description="Compiles tools classes.">
<compile srcdir="${src.dir}" destdir="${build.dir}/classes/java"
includeantruntime="true" />
<compile srcdir="${src.dir}" destdir="${build.dir}/classes/java" includeantruntime="true">
<classpath refid="classpath"/>
</compile>
<copy todir="${build.dir}/classes/java">
<fileset dir="${src.dir}" excludes="**/*.java" />
</copy>

18
lucene/tools/custom-tasks.xml
View File

@ -5,19 +5,23 @@
for standalone use.
</description>
<target name="load-custom-tasks" unless="custom-tasks.loaded">
<dirname file="${ant.file.custom-tasks}" property="custom-tasks.dir"/>
<taskdef resource="lucene-solr.antlib.xml">
<classpath>
<pathelement location="${custom-tasks.dir}/../build/tools/classes/java" />
<fileset dir="${custom-tasks.dir}/lib" includes="asm-debug-all-4.0.jar" />
</classpath>
</taskdef>
<property name="custom-tasks.loaded" value="true"/>
</target>
<macrodef name="license-check-macro">
<attribute name="dir" />
<element name="additional-excludes" optional="true" />
<element name="additional-filters" optional="true" />
<sequential>
<!-- LICENSE and NOTICE verification macro. -->
<dirname file="${ant.file.custom-tasks}" property="custom-tasks.dir"/>
<taskdef resource="lucene-solr.antlib.xml">
<classpath>
<pathelement location="${custom-tasks.dir}/../build/tools/classes/java" />
</classpath>
</taskdef>
<echo>License check under: @{dir}</echo>
<licenses>
<fileset dir="@{dir}">

20
lucene/tools/forbiddenApis/commons-io.txt Normal file
View File

@ -0,0 +1,20 @@
# These methods and classes from commons-io should not be used by Solr classes (unsafe, no charset,...):
org.apache.commons.io.IOUtils#copy(java.io.InputStream,java.io.Writer)
org.apache.commons.io.IOUtils#copy(java.io.Reader,java.io.OutputStream)
org.apache.commons.io.IOUtils#readLines(java.io.InputStream)
org.apache.commons.io.IOUtils#toByteArray(java.io.Reader)
org.apache.commons.io.IOUtils#toByteArray(java.lang.String)
org.apache.commons.io.IOUtils#toCharArray(java.io.InputStream)
org.apache.commons.io.IOUtils#toInputStream(java.lang.CharSequence)
org.apache.commons.io.IOUtils#toInputStream(java.lang.String)
org.apache.commons.io.IOUtils#toString(byte[])
org.apache.commons.io.IOUtils#toString(java.io.InputStream)
org.apache.commons.io.IOUtils#toString(java.net.URI)
org.apache.commons.io.IOUtils#toString(java.net.URL)
org.apache.commons.io.IOUtils#write(byte[],java.io.Writer)
org.apache.commons.io.IOUtils#write(char[],java.io.OutputStream)
org.apache.commons.io.IOUtils#write(java.lang.CharSequence,java.io.OutputStream)
org.apache.commons.io.IOUtils#write(java.lang.StringBuffer,java.io.OutputStream)
org.apache.commons.io.IOUtils#write(java.lang.String,java.io.OutputStream)
org.apache.commons.io.IOUtils#writeLines(java.util.Collection,java.lang.String,java.io.OutputStream)

14
lucene/tools/forbiddenApis/jdk.txt Normal file
View File

@ -0,0 +1,14 @@
# These methods and classes should not be used by Lucene classes (unsafe, no charset,...):
java.lang.String#<init>(byte[])
java.lang.String#<init>(byte[],int)
java.lang.String#<init>(byte[],int,int)
java.lang.String#<init>(byte[],int,int)
java.lang.String#getBytes()
java.lang.String#toLowerCase()
java.lang.String#toUpperCase()
java.io.FileReader
java.io.FileWriter
java.io.InputStreamReader#<init>(java.io.InputStream)
java.io.OutputStreamWriter#<init>(java.io.OutputStream)

8
lucene/tools/ivy.xml
View File

@ -18,4 +18,12 @@
-->
<ivy-module version="2.0">
<info organisation="org.apache.lucene" module="core-tools"/>
<dependencies>
<!--
We use the asm-debug-all library, as the as-all has no generics.
It's not really debug, just not "stripped":
-->
<dependency org="org.ow2.asm" name="asm-debug-all" rev="4.0" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>

1
lucene/tools/lib/asm-debug-all-4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2340f4db0d1a57ba3a430597c42875c827a4cb69

29
lucene/tools/lib/asm-debug-all-LICENSE-BSD_LIKE.txt Normal file
View File

@ -0,0 +1,29 @@
Copyright (c) 2000-2011 INRIA, France Telecom
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holders nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.

2
lucene/tools/lib/asm-debug-all-NOTICE.txt Normal file
View File

@ -0,0 +1,2 @@
ASM - Lightweight Java Bytecode Manipulation Framework
Copyright © 1999-2012, OW2 Consortium

3
lucene/tools/src/java/lucene-solr.antlib.xml
View File

@ -18,4 +18,7 @@
<taskdef
name="licenses"
classname="org.apache.lucene.validation.LicenseCheckTask" />
<taskdef
name="forbidden-apis"
classname="org.apache.lucene.validation.ForbiddenApisCheckTask" />
</antlib>

308
lucene/tools/src/java/org/apache/lucene/validation/ForbiddenApisCheckTask.java Normal file
View File

@ -0,0 +1,308 @@
package org.apache.lucene.validation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.Label;
import org.objectweb.asm.ClassVisitor;
import org.objectweb.asm.MethodVisitor;
import org.objectweb.asm.Opcodes;
import org.objectweb.asm.Type;
import org.objectweb.asm.commons.Method;
import org.objectweb.asm.tree.ClassNode;
import org.objectweb.asm.tree.MethodNode;
import org.apache.tools.ant.AntClassLoader;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.types.Path;
import org.apache.tools.ant.types.FileSet;
import org.apache.tools.ant.types.Reference;
import org.apache.tools.ant.types.Resource;
import org.apache.tools.ant.types.ResourceCollection;
import org.apache.tools.ant.types.resources.FileResource;
import org.apache.tools.ant.types.resources.Resources;
import org.apache.tools.ant.types.resources.FileResource;
import org.apache.tools.ant.types.resources.StringResource;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.BufferedReader;
import java.io.Reader;
import java.io.File;
import java.io.StringReader;
import java.util.Arrays;
import java.util.Formatter;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* Task to check if a set of class files contains calls to forbidden methods
* from a given classpath and list of methods (either inline or as pointer to files).
*/
public class ForbiddenApisCheckTask extends Task {
private final Resources classFiles = new Resources();
private final Resources apiSignatures = new Resources();
private Path classpath = null;
private final Map<String,ClassNode> classCache = new HashMap<String,ClassNode>();
private final Map<String,String> forbiddenMethods = new HashMap<String,String>();
private final Map<String,String> forbiddenClasses = new HashMap<String,String>();
/** Adds the method signature to the list of disallowed methods. The Signature is checked against the given ClassLoader. */
private void addSignature(ClassLoader loader, String signature) throws BuildException {
final int p = signature.indexOf('#');
final String clazz;
final Method dummy;
if (p >= 0) {
clazz = signature.substring(0, p);
// we ignore the return type, its just to match easier (so return type is void):
dummy = Method.getMethod("void " + signature.substring(p+1), true);
} else {
clazz = signature;
dummy = null;
}
// check class & method signature, if it is really existent (in classpath), but we don't really load the class into JVM:
try {
ClassNode c = classCache.get(clazz);
if (c == null) {
final ClassReader reader;
if (loader != null) {
final InputStream in = loader.getResourceAsStream(clazz.replace('.', '/') + ".class");
if (in == null) {
throw new BuildException("Loading of class " + clazz + " failed: Not found");
}
try {
reader = new ClassReader(in);
} finally {
in.close();
}
} else {
// load from build classpath
reader = new ClassReader(clazz);
}
reader.accept(c = new ClassNode(Opcodes.ASM4), ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
classCache.put(clazz, c);
}
if (dummy != null) {
// list all methods with this signature:
boolean found = false;
for (final MethodNode mn : c.methods) {
if (mn.name.equals(dummy.getName()) && Arrays.equals(Type.getArgumentTypes(mn.desc), dummy.getArgumentTypes())) {
found = true;
forbiddenMethods.put(c.name + '\000' + new Method(mn.name, mn.desc), signature);
// don't break when found, as there may be more covariant overrides!
}
}
if (!found)
throw new BuildException("No method found with following signature: " + signature);
} else {
// only add the signature as class name
forbiddenClasses.put(c.name, signature);
}
} catch (IOException e) {
throw new BuildException("Loading of class " + clazz + " failed.", e);
}
}
/** Parses a class given as Resource and checks for valid method invocations */
private int checkClass(final Resource res) throws IOException {
final InputStream stream = res.getInputStream();
try {
final int[] violations = new int[1];
new ClassReader(stream).accept(new ClassVisitor(Opcodes.ASM4) {
String className = null, source = null;
@Override
public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
// save class name in source code format:
this.className = Type.getObjectType(name).getClassName();
}
@Override
public void visitSource(String source, String debug) {
this.source = source;
}
@Override
public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
return new MethodVisitor(Opcodes.ASM4) {
private int lineNo = -1;
@Override
public void visitMethodInsn(int opcode, String owner, String name, String desc) {
boolean found = false;
String printout = forbiddenClasses.get(owner);
if (printout != null) {
found = true;
log("Forbidden class use: " + printout, Project.MSG_ERR);
} else {
printout = forbiddenMethods.get(owner + '\000' + new Method(name, desc));
if (printout != null) {
found = true;
log("Forbidden method invocation: " + printout, Project.MSG_ERR);
}
}
if (found) {
violations[0]++;
final StringBuilder sb = new StringBuilder(" in ").append(className);
if (source != null && lineNo >= 0) {
new Formatter(sb, Locale.ENGLISH).format(" (%s:%d)", source, lineNo).flush();
}
log(sb.toString(), Project.MSG_ERR);
}
}
@Override
public void visitLineNumber(int lineNo, Label start) {
this.lineNo = lineNo;
}
};
}
}, ClassReader.SKIP_FRAMES);
return violations[0];
} finally {
stream.close();
}
}
/** Reads a list of method signatures. Closes the Reader when done (on Exception, too)! */
private void parseApiFile(ClassLoader loader, Reader reader) throws IOException {
final BufferedReader r = new BufferedReader(reader);
try {
String line;
while ((line = r.readLine()) != null) {
line = line.trim();
if (line.length() == 0 || line.startsWith("#"))
continue;
addSignature(loader, line);
}
} finally {
r.close();
}
}
@Override
public void execute() throws BuildException {
AntClassLoader loader = null;
try {
if (classpath != null) {
classpath.setProject(getProject());
loader = getProject().createClassLoader(classpath);
}
classFiles.setProject(getProject());
try {
@SuppressWarnings("unchecked")
Iterator<Resource> iter = (Iterator<Resource>) apiSignatures.iterator();
while (iter.hasNext()) {
final Resource r = iter.next();
if (!r.isExists()) {
throw new BuildException("Resource does not exist: " + r.getName());
}
if (r instanceof StringResource) {
parseApiFile(loader, new StringReader(((StringResource) r).getValue()));
} else {
parseApiFile(loader, new InputStreamReader(r.getInputStream(), "UTF-8"));
}
}
} catch (IOException ioe) {
throw new BuildException("IO problem while reading files with API signatures.", ioe);
}
long start = System.currentTimeMillis();
int checked = 0;
int errors = 0;
@SuppressWarnings("unchecked")
Iterator<Resource> iter = (Iterator<Resource>) classFiles.iterator();
while (iter.hasNext()) {
final Resource r = iter.next();
if (!r.isExists()) {
throw new BuildException("Class file does not exist: " + r.getName());
}
try {
errors += checkClass(r);
} catch (IOException ioe) {
throw new BuildException("IO problem while reading class file " + r.getName(), ioe);
}
checked++;
}
log(String.format(Locale.ENGLISH,
"Scanned %d class file(s) for forbidden method invocations (in %.2fs), %d error(s).",
checked, (System.currentTimeMillis() - start) / 1000.0, errors),
errors > 0 ? Project.MSG_ERR : Project.MSG_INFO);
if (errors > 0) {
throw new BuildException("Check for forbidden method calls failed, see log.");
}
} finally {
if (loader != null) loader.cleanup();
}
}
/** Set of class files to check */
public void add(ResourceCollection rc) {
classFiles.add(rc);
}
/** A file with method signatures apiFile= attribute */
public void setApiFile(File file) {
apiSignatures.add(new FileResource(getProject(), file));
}
/** Set of files with method signatures as <apiFileSet/> nested element */
public FileSet createApiFileSet() {
final FileSet fs = new FileSet();
fs.setProject(getProject());
apiSignatures.add(fs);
return fs;
}
/** Support for API signatures list as nested text */
public void addText(String text) {
apiSignatures.add(new StringResource(getProject(), text));
}
/** Classpath as classpath= attribute */
public void setClasspath(Path classpath) {
createClasspath().append(classpath);
}
/** Classpath as classpathRef= attribute */
public void setClasspathRef(Reference r) {
createClasspath().setRefid(r);
}
/** Classpath as <classpath/> nested element */
public Path createClasspath() {
if (this.classpath == null) {
this.classpath = new Path(getProject());
}
return this.classpath.createPath();
}
}

17
solr/build.xml
View File

@ -170,8 +170,10 @@
</ant>
</target>
<!-- Validation (license/ notice checks). -->
<target name="validate" depends="compile-tools,resolve" description="Validate legal stuff.">
<!-- Validation (license/notice/api checks). -->
<target name="validate" depends="check-licenses,check-forbidden-apis" description="Validate stuff." />
<target name="check-licenses" depends="compile-tools,resolve,load-custom-tasks" description="Validate license stuff.">
<license-check-macro dir="${basedir}">
<additional-excludes>
<!-- Exclude start.jar only (it'd be weird to have a license file there?) -->
@ -188,6 +190,17 @@
</license-check-macro>
</target>
<target name="check-forbidden-apis" depends="compile-tools,compile-test,load-custom-tasks" description="Check forbidden API calls in compiled class files.">
<forbidden-apis>
<classpath refid="classpath"/>
<apiFileSet dir="${custom-tasks.dir}/forbiddenApis">
<include name="jdk.txt" />
<include name="commons-io.txt" />
</apiFileSet>
<fileset dir="${basedir}" includes="**/build/**/*.class" />
</forbidden-apis>
</target>
<!-- rat sources -->
<target name="rat-sources">
<sequential>