diff --git a/lucene/build.xml b/lucene/build.xml index 91d3d3ce84c..99e2deae8f4 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -169,11 +169,19 @@ - - + + + + + + + + + + diff --git a/lucene/tools/build.xml b/lucene/tools/build.xml index ab07276a069..106c2da6f19 100644 --- a/lucene/tools/build.xml +++ b/lucene/tools/build.xml @@ -24,7 +24,11 @@ - + + + + + - + + + diff --git a/lucene/tools/custom-tasks.xml b/lucene/tools/custom-tasks.xml index 3391ab5ac7f..55f8f4f3bc6 100644 --- a/lucene/tools/custom-tasks.xml +++ b/lucene/tools/custom-tasks.xml @@ -5,19 +5,23 @@ for standalone use. + + + + + + + + + + + - - - - - - - License check under: @{dir} diff --git a/lucene/tools/forbiddenApis/commons-io.txt b/lucene/tools/forbiddenApis/commons-io.txt new file mode 100644 index 00000000000..d8e089c931d --- /dev/null +++ b/lucene/tools/forbiddenApis/commons-io.txt @@ -0,0 +1,20 @@ +# These methods and classes from commons-io should not be used by Solr classes (unsafe, no charset,...): + +org.apache.commons.io.IOUtils#copy(java.io.InputStream,java.io.Writer) +org.apache.commons.io.IOUtils#copy(java.io.Reader,java.io.OutputStream) +org.apache.commons.io.IOUtils#readLines(java.io.InputStream) +org.apache.commons.io.IOUtils#toByteArray(java.io.Reader) +org.apache.commons.io.IOUtils#toByteArray(java.lang.String) +org.apache.commons.io.IOUtils#toCharArray(java.io.InputStream) +org.apache.commons.io.IOUtils#toInputStream(java.lang.CharSequence) +org.apache.commons.io.IOUtils#toInputStream(java.lang.String) +org.apache.commons.io.IOUtils#toString(byte[]) +org.apache.commons.io.IOUtils#toString(java.io.InputStream) +org.apache.commons.io.IOUtils#toString(java.net.URI) +org.apache.commons.io.IOUtils#toString(java.net.URL) +org.apache.commons.io.IOUtils#write(byte[],java.io.Writer) +org.apache.commons.io.IOUtils#write(char[],java.io.OutputStream) +org.apache.commons.io.IOUtils#write(java.lang.CharSequence,java.io.OutputStream) +org.apache.commons.io.IOUtils#write(java.lang.StringBuffer,java.io.OutputStream) +org.apache.commons.io.IOUtils#write(java.lang.String,java.io.OutputStream) +org.apache.commons.io.IOUtils#writeLines(java.util.Collection,java.lang.String,java.io.OutputStream) diff --git a/lucene/tools/forbiddenApis/jdk.txt b/lucene/tools/forbiddenApis/jdk.txt new file mode 100644 index 00000000000..a1f0815bfb5 --- /dev/null +++ b/lucene/tools/forbiddenApis/jdk.txt @@ -0,0 +1,14 @@ +# These methods and classes should not be used by Lucene classes (unsafe, no charset,...): + +java.lang.String#(byte[]) +java.lang.String#(byte[],int) +java.lang.String#(byte[],int,int) +java.lang.String#(byte[],int,int) +java.lang.String#getBytes() +java.lang.String#toLowerCase() +java.lang.String#toUpperCase() + +java.io.FileReader +java.io.FileWriter +java.io.InputStreamReader#(java.io.InputStream) +java.io.OutputStreamWriter#(java.io.OutputStream) diff --git a/lucene/tools/ivy.xml b/lucene/tools/ivy.xml index 6525f67f041..86fe27988c6 100644 --- a/lucene/tools/ivy.xml +++ b/lucene/tools/ivy.xml @@ -18,4 +18,12 @@ --> + + + + + diff --git a/lucene/tools/lib/asm-debug-all-4.0.jar.sha1 b/lucene/tools/lib/asm-debug-all-4.0.jar.sha1 new file mode 100644 index 00000000000..c678c895ce2 --- /dev/null +++ b/lucene/tools/lib/asm-debug-all-4.0.jar.sha1 @@ -0,0 +1 @@ +2340f4db0d1a57ba3a430597c42875c827a4cb69 diff --git a/lucene/tools/lib/asm-debug-all-LICENSE-BSD_LIKE.txt b/lucene/tools/lib/asm-debug-all-LICENSE-BSD_LIKE.txt new file mode 100644 index 00000000000..c5aba7be471 --- /dev/null +++ b/lucene/tools/lib/asm-debug-all-LICENSE-BSD_LIKE.txt @@ -0,0 +1,29 @@ +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lucene/tools/lib/asm-debug-all-NOTICE.txt b/lucene/tools/lib/asm-debug-all-NOTICE.txt new file mode 100644 index 00000000000..f6df5a6fe78 --- /dev/null +++ b/lucene/tools/lib/asm-debug-all-NOTICE.txt @@ -0,0 +1,2 @@ +ASM - Lightweight Java Bytecode Manipulation Framework +Copyright © 1999-2012, OW2 Consortium diff --git a/lucene/tools/src/java/lucene-solr.antlib.xml b/lucene/tools/src/java/lucene-solr.antlib.xml index 6ab57c6695a..f18d8a3287b 100644 --- a/lucene/tools/src/java/lucene-solr.antlib.xml +++ b/lucene/tools/src/java/lucene-solr.antlib.xml @@ -18,4 +18,7 @@ + diff --git a/lucene/tools/src/java/org/apache/lucene/validation/ForbiddenApisCheckTask.java b/lucene/tools/src/java/org/apache/lucene/validation/ForbiddenApisCheckTask.java new file mode 100644 index 00000000000..2012a055445 --- /dev/null +++ b/lucene/tools/src/java/org/apache/lucene/validation/ForbiddenApisCheckTask.java @@ -0,0 +1,308 @@ +package org.apache.lucene.validation; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.Label; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.commons.Method; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.MethodNode; + +import org.apache.tools.ant.AntClassLoader; +import org.apache.tools.ant.BuildException; +import org.apache.tools.ant.Project; +import org.apache.tools.ant.Task; +import org.apache.tools.ant.types.Path; +import org.apache.tools.ant.types.FileSet; +import org.apache.tools.ant.types.Reference; +import org.apache.tools.ant.types.Resource; +import org.apache.tools.ant.types.ResourceCollection; +import org.apache.tools.ant.types.resources.FileResource; +import org.apache.tools.ant.types.resources.Resources; +import org.apache.tools.ant.types.resources.FileResource; +import org.apache.tools.ant.types.resources.StringResource; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.BufferedReader; +import java.io.Reader; +import java.io.File; +import java.io.StringReader; +import java.util.Arrays; +import java.util.Formatter; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Task to check if a set of class files contains calls to forbidden methods + * from a given classpath and list of methods (either inline or as pointer to files). + */ +public class ForbiddenApisCheckTask extends Task { + + private final Resources classFiles = new Resources(); + private final Resources apiSignatures = new Resources(); + private Path classpath = null; + + private final Map classCache = new HashMap(); + private final Map forbiddenMethods = new HashMap(); + private final Map forbiddenClasses = new HashMap(); + + /** Adds the method signature to the list of disallowed methods. The Signature is checked against the given ClassLoader. */ + private void addSignature(ClassLoader loader, String signature) throws BuildException { + final int p = signature.indexOf('#'); + final String clazz; + final Method dummy; + if (p >= 0) { + clazz = signature.substring(0, p); + // we ignore the return type, its just to match easier (so return type is void): + dummy = Method.getMethod("void " + signature.substring(p+1), true); + } else { + clazz = signature; + dummy = null; + } + // check class & method signature, if it is really existent (in classpath), but we don't really load the class into JVM: + try { + ClassNode c = classCache.get(clazz); + if (c == null) { + final ClassReader reader; + if (loader != null) { + final InputStream in = loader.getResourceAsStream(clazz.replace('.', '/') + ".class"); + if (in == null) { + throw new BuildException("Loading of class " + clazz + " failed: Not found"); + } + try { + reader = new ClassReader(in); + } finally { + in.close(); + } + } else { + // load from build classpath + reader = new ClassReader(clazz); + } + reader.accept(c = new ClassNode(Opcodes.ASM4), ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); + classCache.put(clazz, c); + } + if (dummy != null) { + // list all methods with this signature: + boolean found = false; + for (final MethodNode mn : c.methods) { + if (mn.name.equals(dummy.getName()) && Arrays.equals(Type.getArgumentTypes(mn.desc), dummy.getArgumentTypes())) { + found = true; + forbiddenMethods.put(c.name + '\000' + new Method(mn.name, mn.desc), signature); + // don't break when found, as there may be more covariant overrides! + } + } + if (!found) + throw new BuildException("No method found with following signature: " + signature); + } else { + // only add the signature as class name + forbiddenClasses.put(c.name, signature); + } + } catch (IOException e) { + throw new BuildException("Loading of class " + clazz + " failed.", e); + } + } + + /** Parses a class given as Resource and checks for valid method invocations */ + private int checkClass(final Resource res) throws IOException { + final InputStream stream = res.getInputStream(); + try { + final int[] violations = new int[1]; + new ClassReader(stream).accept(new ClassVisitor(Opcodes.ASM4) { + String className = null, source = null; + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + // save class name in source code format: + this.className = Type.getObjectType(name).getClassName(); + } + + @Override + public void visitSource(String source, String debug) { + this.source = source; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + return new MethodVisitor(Opcodes.ASM4) { + private int lineNo = -1; + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc) { + boolean found = false; + String printout = forbiddenClasses.get(owner); + if (printout != null) { + found = true; + log("Forbidden class use: " + printout, Project.MSG_ERR); + } else { + printout = forbiddenMethods.get(owner + '\000' + new Method(name, desc)); + if (printout != null) { + found = true; + log("Forbidden method invocation: " + printout, Project.MSG_ERR); + } + } + if (found) { + violations[0]++; + final StringBuilder sb = new StringBuilder(" in ").append(className); + if (source != null && lineNo >= 0) { + new Formatter(sb, Locale.ENGLISH).format(" (%s:%d)", source, lineNo).flush(); + } + log(sb.toString(), Project.MSG_ERR); + } + } + + @Override + public void visitLineNumber(int lineNo, Label start) { + this.lineNo = lineNo; + } + }; + } + }, ClassReader.SKIP_FRAMES); + return violations[0]; + } finally { + stream.close(); + } + } + + /** Reads a list of method signatures. Closes the Reader when done (on Exception, too)! */ + private void parseApiFile(ClassLoader loader, Reader reader) throws IOException { + final BufferedReader r = new BufferedReader(reader); + try { + String line; + while ((line = r.readLine()) != null) { + line = line.trim(); + if (line.length() == 0 || line.startsWith("#")) + continue; + addSignature(loader, line); + } + } finally { + r.close(); + } + } + + @Override + public void execute() throws BuildException { + AntClassLoader loader = null; + try { + if (classpath != null) { + classpath.setProject(getProject()); + loader = getProject().createClassLoader(classpath); + } + classFiles.setProject(getProject()); + + try { + @SuppressWarnings("unchecked") + Iterator iter = (Iterator) apiSignatures.iterator(); + while (iter.hasNext()) { + final Resource r = iter.next(); + if (!r.isExists()) { + throw new BuildException("Resource does not exist: " + r.getName()); + } + if (r instanceof StringResource) { + parseApiFile(loader, new StringReader(((StringResource) r).getValue())); + } else { + parseApiFile(loader, new InputStreamReader(r.getInputStream(), "UTF-8")); + } + } + } catch (IOException ioe) { + throw new BuildException("IO problem while reading files with API signatures.", ioe); + } + + long start = System.currentTimeMillis(); + + int checked = 0; + int errors = 0; + @SuppressWarnings("unchecked") + Iterator iter = (Iterator) classFiles.iterator(); + while (iter.hasNext()) { + final Resource r = iter.next(); + if (!r.isExists()) { + throw new BuildException("Class file does not exist: " + r.getName()); + } + + try { + errors += checkClass(r); + } catch (IOException ioe) { + throw new BuildException("IO problem while reading class file " + r.getName(), ioe); + } + checked++; + } + + log(String.format(Locale.ENGLISH, + "Scanned %d class file(s) for forbidden method invocations (in %.2fs), %d error(s).", + checked, (System.currentTimeMillis() - start) / 1000.0, errors), + errors > 0 ? Project.MSG_ERR : Project.MSG_INFO); + + if (errors > 0) { + throw new BuildException("Check for forbidden method calls failed, see log."); + } + } finally { + if (loader != null) loader.cleanup(); + } + } + + /** Set of class files to check */ + public void add(ResourceCollection rc) { + classFiles.add(rc); + } + + /** A file with method signatures apiFile= attribute */ + public void setApiFile(File file) { + apiSignatures.add(new FileResource(getProject(), file)); + } + + /** Set of files with method signatures as nested element */ + public FileSet createApiFileSet() { + final FileSet fs = new FileSet(); + fs.setProject(getProject()); + apiSignatures.add(fs); + return fs; + } + + /** Support for API signatures list as nested text */ + public void addText(String text) { + apiSignatures.add(new StringResource(getProject(), text)); + } + + /** Classpath as classpath= attribute */ + public void setClasspath(Path classpath) { + createClasspath().append(classpath); + } + + /** Classpath as classpathRef= attribute */ + public void setClasspathRef(Reference r) { + createClasspath().setRefid(r); + } + + /** Classpath as nested element */ + public Path createClasspath() { + if (this.classpath == null) { + this.classpath = new Path(getProject()); + } + return this.classpath.createPath(); + } + +} diff --git a/solr/build.xml b/solr/build.xml index 4f0c9673882..d1715d4861a 100644 --- a/solr/build.xml +++ b/solr/build.xml @@ -170,8 +170,10 @@ - - + + + + @@ -187,6 +189,17 @@ + + + + + + + + + + +