LUCENE-3774: Optimized and streamlined license and notice file validation

by refactoring the build task into an ANT task and modifying build scripts
to perform top-level checks. (Dawid Weiss, Steve Rowe, Robert Muir)

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1243527 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Dawid Weiss 2012-02-13 14:12:59 +00:00
parent 337270d1ad
commit 087f1e3126
26 changed files with 362 additions and 505 deletions

View File

@ -27,6 +27,7 @@
</subant>
</sequential>
</target>
<target name="javadocs" description="Generate Lucene and Solr javadocs">
<sequential>
<subant target="javadocs" inheritall="false" failonerror="true">
@ -36,6 +37,7 @@
</subant>
</sequential>
</target>
<target name="validate" description="Validate dependencies, licenses, etc.">
<sequential><subant target="validate" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml" />
@ -43,6 +45,7 @@
<fileset dir="solr" includes="build.xml" />
</subant></sequential>
</target>
<target name="compile" description="Compile Lucene and Solr">
<sequential>

View File

@ -641,6 +641,10 @@ New features
Optimizations
* LUCENE-3774: Optimized and streamlined license and notice file validation
by refactoring the build task into an ANT task and modifying build scripts
to perform top-level checks. (Dawid Weiss, Steve Rowe, Robert Muir)
* LUCENE-2588: Don't store unnecessary suffixes when writing the terms
index, saving RAM in IndexReader; change default terms index
interval from 128 to 32, because the terms index now requires much

View File

@ -44,6 +44,7 @@
<propertyset refid="uptodate.and.compiled.properties"/>
</ant>
</target>
<target name="test" depends="test-core, test-contrib, test-backwards"
description="Runs all unit tests (core, contrib and back-compat)"
/>
@ -172,6 +173,11 @@
</clover-report>
</target>
<!-- Validate once from top-level. -->
<target name="validate" depends="compile-tools" description="Validate legal stuff.">
<license-check-macro dir="${basedir}" />
</target>
<!-- ================================================================== -->
<!-- D O C U M E N T A T I O N -->
<!-- ================================================================== -->

View File

@ -62,10 +62,6 @@
<fileset dir="${common.dir}/lib" includes="ant-*.jar"/>
</path>
<path id="tools.runtime.classpath">
<pathelement location="${common.dir}/build/tools/classes/java"/>
</path>
<path id="maven-ant-tasks.classpath">
<fileset dir="${common.dir}/lib">
<include name="maven-ant-tasks-*.jar"/>
@ -204,16 +200,17 @@
<property name="clover.db.dir" location="${common.dir}/build/test/clover/db"/>
<property name="clover.report.dir" location="${common.dir}/build/test/clover/reports"/>
<available
<available
property="clover.present"
classname="com.cenqua.clover.tasks.CloverReportTask"
/>
<condition property="clover.enabled">
<and>
<isset property="run.clover"/>
<isset property="clover.present"/>
</and>
</condition>
/>
<condition property="clover.enabled">
<and>
<isset property="run.clover"/>
<isset property="clover.present"/>
</and>
</condition>
<propertyset id="uptodate.and.compiled.properties" dynamic="true">
<propertyref regex=".*\.uptodate$$"/>
@ -225,6 +222,9 @@
excludes="**/pom.xml,**/*.iml,site/build/"
/>
<!-- Import custom ANT tasks. -->
<import file="${common.dir}/tools/custom-tasks.xml" />
<target name="clean"
description="Removes contents of build and dist directories">
<delete dir="${build.dir}"/>
@ -678,11 +678,11 @@
<fail if="tests.failed">Tests failed!</fail>
</sequential>
</macrodef>
<target name="test" depends="compile-test,validate-lucene,junit-mkdir,junit-sequential,junit-parallel" description="Runs unit tests"/>
<target name="test" depends="compile-test,validate,junit-mkdir,junit-sequential,junit-parallel" description="Runs unit tests"/>
<target name="junit-mkdir">
<mkdir dir="${junit.output.dir}"/>
<mkdir dir="${junit.output.dir}"/>
</target>
<target name="junit-sequential" if="tests.sequential">
@ -820,12 +820,14 @@
<attribute name="destdir"/>
<attribute name="javac.source" default="${javac.source}"/>
<attribute name="javac.target" default="${javac.target}"/>
<attribute name="includeantruntime" default="${javac.includeAntRuntime}" />
<element name="nested" implicit="yes" optional="yes"/>
<sequential>
<mkdir dir="@{destdir}"/>
<javac
includeAntRuntime="${javac.includeAntRuntime}"
includeAntRuntime="@{includeantruntime}"
encoding="${build.encoding}"
srcdir="@{srcdir}"
destdir="@{destdir}"
@ -955,26 +957,9 @@
<!-- VALIDATION work -->
<target name="check-legal-lucene" depends="compile-tools">
<java classname="org.apache.lucene.validation.DependencyChecker" failonerror="true" fork="true">
<classpath>
<path refid="tools.runtime.classpath" />
</classpath>
<!-- TODO: it might be better to just automatically find all directories that contain jar files, but that could take a
long time. This should be faster, but we could miss a directory
-->
<!-- Lucene -->
<arg value="-c" />
<arg value="${common.dir}/lib" />
</java>
</target>
<target name="check-legal" depends="check-legal-lucene"/>
<target name="validate-lucene" depends="check-legal-lucene" unless="validated-lucene"/>
<!-- Generic placeholder target for if we add other validation tasks -->
<target name="validate" depends="validate-lucene"/>
<target name="validate" depends="compile-tools">
</target>
<property name="svn.export.dir" location="${build.dir}/svn-export"/>
<macrodef name="svn-export-source"
@ -1096,5 +1081,4 @@
</scp>
</sequential>
</macrodef>
</project>

View File

@ -24,7 +24,7 @@
<property name="build.dir" location="../../build/contrib/${ant.project.name}"/>
<property name="dist.dir" location="../../dist/contrib/${ant.project.name}"/>
<property name="maven.dist.dir" location="../../dist/maven"/>
<import file="../common-build.xml"/>
<available property="contrib.has.tests" type="dir" file="src/test" />

View File

@ -0,0 +1,4 @@
cpptasks for Apache Ant
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).

View File

@ -0,0 +1,6 @@
Apache Ant
Copyright 1999-2008 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).

View File

@ -31,9 +31,13 @@
classpath reference when compiling, and to not attempt to copy
non-existent resource files to the build output directory.
-->
<target name="compile-core" depends="init" description="Compiles tools classes">
<compile srcdir="${src.dir}" destdir="${build.dir}/classes/java"/>
<target name="compile-core" depends="init" description="Compiles tools classes.">
<compile srcdir="${src.dir}" destdir="${build.dir}/classes/java"
includeantruntime="true" />
<copy todir="${build.dir}/classes/java">
<fileset dir="${src.dir}" excludes="**/*.java" />
</copy>
</target>
<target name="javadocs"/> <!-- to make common-build.xml happy -->
</project>

View File

@ -0,0 +1,59 @@
<project name="custom-tasks">
<description>
This file is designed for importing into a main build file, and not intended
for standalone use.
</description>
<macrodef name="license-check-macro">
<attribute name="dir" />
<element name="additional-excludes" optional="true" />
<element name="additional-filters" optional="true" />
<sequential>
<!-- LICENSE and NOTICE verification macro. -->
<dirname file="${ant.file.custom-tasks}" property="custom-tasks.dir"/>
<taskdef resource="lucene-solr.antlib.xml">
<classpath>
<pathelement location="${custom-tasks.dir}/../build/tools/classes/java" />
</classpath>
</taskdef>
<echo>License check under: @{dir}</echo>
<licenses>
<fileset dir="@{dir}">
<include name="**/*.jar" />
<!-- Speed up scanning a bit. -->
<exclude name="**/.git/**" />
<exclude name="**/.svn/**" />
<exclude name="**/bin/**" />
<exclude name="**/build/**" />
<exclude name="**/dist/**" />
<exclude name="**/src/**" />
<additional-excludes />
</fileset>
<licenseMapper>
<filtermapper id="license-mapper-defaults">
<!-- Normalize input paths. -->
<replacestring from="\" to="/" />
<replaceregex pattern="\.jar$" replace="" flags="gi" />
<!-- Some typical snapshot/minimalized JAR suffixes. -->
<replaceregex pattern="-min$" replace="" flags="gi" />
<replaceregex pattern="SNAPSHOT$" replace="" flags="gi" />
<!-- Non-typical version patterns. -->
<additional-filters />
<replaceregex pattern="/xercesImpl([^/]+)$" replace="/xercesImpl" flags="gi" />
<replaceregex pattern="/commons-csv-([^/]+)$" replace="/commons-csv" flags="gi" />
<replaceregex pattern="/(bcmail|bcprov)-([^/]+)$" replace="/\1" flags="gi" />
<replaceregex pattern="/slf4j-([^/]+)$" replace="/slf4j" flags="gi" />
<!-- Typical version patterns. -->
<replaceregex pattern="\-(r)?([0-9\-\_\.])+(b(eta)?([0-9\-\.])*)?$" replace="" flags="gi" />
</filtermapper>
</licenseMapper>
</licenses>
</sequential>
</macrodef>
</project>

View File

@ -0,0 +1,5 @@
<antlib>
<taskdef
name="licenses"
classname="org.apache.lucene.validation.LicenseCheckTask" />
</antlib>

View File

@ -1,282 +0,0 @@
package org.apache.lucene.validation;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.FileFilter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
*
*
**/
public class DependencyChecker {
private static Set<String> excludes = new HashSet<String>();
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
static {
//Collections.addAll(excludes, );
}
public static void main(String[] args) throws IOException {
String dumpFile = null;
List<String> dirs = new ArrayList<String>();
for (int i = 0; i < args.length; i++) {
if (args[i].equalsIgnoreCase("--dump") || args[i].equalsIgnoreCase("-d")) {
dumpFile = args[++i];
} else if (args[i].equalsIgnoreCase("--check") || args[i].equalsIgnoreCase("-c")) {
dirs.add(args[++i]);
}
}
FileWriter writer = null;
boolean dump = false;
if (dumpFile != null) {
File out = new File(dumpFile);
System.out.println("Dumping to " + out);
writer = new FileWriter(out);
dump = true;
}
//TODO: put in NOTICE checks
for (String checkDir : dirs) {
File dir = new File(checkDir);
if (dir.exists()) {
System.out.println("----------------------");
System.out.println("Starting on dir: " + dir);
int numFailed = 0;
File[] list = dir.listFiles();
File[] licFiles = dir.listFiles(new FileFilter() {
public boolean accept(File file) {
return file.getName().indexOf("-LICENSE") != -1 && file.getName().endsWith(".txt");//check for a consistent end, so that we aren't fooled by emacs ~ files or other temp files
}
});
File[] noticeFiles = dir.listFiles(new FileFilter() {
public boolean accept(File file) {
return file.getName().indexOf("-NOTICE") != -1 && file.getName().endsWith(".txt");
}
});
File[] jarFiles = dir.listFiles(new FileFilter() {
public boolean accept(File file) {
return file.getName().endsWith(".jar");
}
});
if (licFiles.length == 0 && jarFiles.length != 0) {
System.out.println("No license files found: " + dir);
numFailed++;
}
if (jarFiles.length != licFiles.length) {
System.out.println("WARNING: There are missing LICENSE files in: " + dir + " Jar file count: " + jarFiles.length + " License Count: " + licFiles.length);
printDiffs(jarFiles, licFiles);
numFailed++;
}
if (jarFiles.length != noticeFiles.length) {
System.out.println("WARNING: There may be missing NOTICE files in: " + dir + ". Note, not all files require a NOTICE. Jar file count: " + jarFiles.length + " Notice Count: " + noticeFiles.length);
//printDiffs(jarFiles, noticeFiles);
}
Map<String, UpdateableInt> licenseNames = new HashMap<String, UpdateableInt>();
for (int i = 0; i < licFiles.length; i++) {
licenseNames.put(licFiles[i].getName(), new UpdateableInt());
}
Map<String, UpdateableInt> noticeNames = new HashMap<String, UpdateableInt>();
for (int i = 0; i < noticeFiles.length; i++) {
noticeNames.put(noticeFiles[i].getName(), new UpdateableInt());
}
for (int i = 0; i < list.length; i++) {
File file = list[i];
String fileName = file.getName();
if (fileName.endsWith(".jar") && excludes.contains(fileName) == false) {
File licFile = getLicenseFile(file, licenseNames);
if (licFile != null && licFile.exists()) {
String licName = licFile.getName();
LicenseType[] types = getLicenseTypes(licName);
if (types != null && types.length > 0) {
for (int j = 0; j < types.length; j++) {
LicenseType type = types[j];
if (dump == true) {
writer.write(file.getName() + "," + type.getDisplay() + LINE_SEPARATOR);
}
if (type.isNoticeRequired()) {
File noticeFile = getNoticeFile(file, noticeNames);
if (noticeFile != null && noticeFile.exists()) {
} else {
System.out.println("!!!!!! Missing NOTICE file for " + file + " and license type: " + type.getDisplay());
if (dump) {
writer.write("Missing NOTICE file for " + file + LINE_SEPARATOR);
}
numFailed++;
}
}
}
} else {
System.out.println("!!!!!! Couldn't determine license type for file: " + file);
if (dump == true) {
writer.write("Invalid license for file: " + file + LINE_SEPARATOR);
}
numFailed++;
}
} else {
System.out.println("!!!!!!! Couldn't get license file for " + file);
if (dump == true) {
writer.write("Couldn't get license file for " + file + LINE_SEPARATOR);
}
numFailed++;
}
}
}
if (dump == true) {
writer.write(LINE_SEPARATOR + LINE_SEPARATOR);
writer.write("Other Licenses (installer, javascript, etc." + LINE_SEPARATOR);
}
if (dump == true) {
for (Map.Entry<String, UpdateableInt> entry : licenseNames.entrySet()) {
if (entry.getValue().theInt == 0) {
LicenseType[] types = getLicenseTypes(entry.getKey());
if (types != null && types.length > 0) {
for (int i = 0; i < types.length; i++) {
writer.write(entry.getKey() + "," + types[i].getDisplay() + LINE_SEPARATOR);
}
} else {
System.out.println("Couldn't determine license for: " + entry.getKey());
}
}
}
}
if (writer != null) {
writer.close();
}
if (numFailed > 0) {
System.out.println("At least one file does not have a license, or it's license name is not in the proper format. See the logs.");
System.exit(-1);
} else {
System.out.println("Found a license for every file in " + dir);
}
} else {
System.out.println("Could not find directory:" + dir);
}
}
}
/**
* Sort the two lists and then print them out for visual comparison
*
* @param left
* @param right
*/
private static void printDiffs(File[] left, File[] right) {
Arrays.sort(left);
Arrays.sort(right);
System.out.println("Left\t\t\tRight");
System.out.println("----------------");
StringBuilder bldr = new StringBuilder();
int i = 0;
for (; i < left.length; i++) {
bldr.append(left[i]).append("\t\t\t");
if (i < right.length) {
bldr.append(right[i]);
}
bldr.append(LINE_SEPARATOR);
}
if (i < right.length) {
for (; i < right.length; i++) {
bldr.append("--- N/A ---\t\t\t").append(right[i]).append(LINE_SEPARATOR);
}
}
System.out.println(bldr.toString());
System.out.println("----------------");
}
private static LicenseType[] getLicenseTypes(String licName) {
LicenseType[] result = new LicenseType[0];
int idx = licName.lastIndexOf("-");
if (idx != -1) {
String licAbbrev = licName.substring(idx + 1, licName.length() - ".txt".length());
String[] lics = licAbbrev.split("__");
result = new LicenseType[lics.length];
for (int j = 0; j < lics.length; j++) {
try {
result[j] = LicenseType.valueOf(lics[j].toUpperCase());
} catch (IllegalArgumentException e) {
System.out.println("Invalid license: " + lics[j].toUpperCase() + " for " + licName);
}
}
}
return result;
}
private static File getLicenseFile(File file, Map<String, UpdateableInt> licenseNames) {
File result = null;
String filename = file.getName();
int length = 0;
for (String licName : licenseNames.keySet()) {
String prefix = licName.substring(0, licName.indexOf("-LICENSE"));
String name = null;
//System.out.println("prefix: " + prefix + " lic name: " + licName);
if (filename.toLowerCase().startsWith(prefix.toLowerCase())) {
result = new File(file.getParentFile(), licName);
UpdateableInt ui = licenseNames.get(licName);
ui.theInt++;
} else {
}
}
//System.out.println("License File: " + result + " for file: " + file);
return result;
}
private static File getNoticeFile(File file, Map<String, UpdateableInt> noticeNames) {
File result = null;
String filename = file.getName();
int length = 0;
for (String noticeName : noticeNames.keySet()) {
String prefix = noticeName.substring(0, noticeName.indexOf("-NOTICE"));
String name = null;
//System.out.println("prefix: " + prefix + " lic name: " + licName);
if (filename.toLowerCase().startsWith(prefix.toLowerCase())) {
result = new File(file.getParentFile(), noticeName);
UpdateableInt ui = noticeNames.get(noticeName);
ui.theInt++;
} else {
}
}
//System.out.println("License File: " + result + " for file: " + file);
return result;
}
}
class UpdateableInt {
public int theInt;
}

View File

@ -0,0 +1,179 @@
package org.apache.lucene.validation;
import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.types.Mapper;
import org.apache.tools.ant.types.Resource;
import org.apache.tools.ant.types.ResourceCollection;
import org.apache.tools.ant.types.resources.FileResource;
import org.apache.tools.ant.types.resources.Resources;
import org.apache.tools.ant.util.FileNameMapper;
/**
* An ANT task that verifies if JAR file have associated <tt>LICENSE</tt>
* and <tt>NOTICE</tt> files.
*/
public class LicenseCheckTask extends Task {
/**
* All JAR files to check.
*/
private Resources jarResources = new Resources();
/**
* License file mapper.
*/
private FileNameMapper licenseMapper;
/**
* A logging level associated with verbose logging.
*/
private int verboseLevel = Project.MSG_VERBOSE;
/**
* Failure flag.
*/
private boolean failures;
/**
* Adds a set of JAR resources to check.
*/
public void add(ResourceCollection rc) {
jarResources.add(rc);
}
/**
* Adds a license mapper.
*/
public void addConfiguredLicenseMapper(Mapper mapper) {
if (licenseMapper != null) {
throw new BuildException("Only one license mapper is allowed.");
}
this.licenseMapper = mapper.getImplementation();
}
public void setVerbose(boolean verbose) {
verboseLevel = (verbose ? Project.MSG_INFO : Project.MSG_VERBOSE);
}
/**
* Execute the task.
*/
@Override
public void execute() throws BuildException {
if (licenseMapper == null) {
throw new BuildException("Expected an embedded <licenseMapper>.");
}
jarResources.setProject(getProject());
processJars();
if (failures) {
throw new BuildException("License check failed. Check the logs.");
}
}
/**
* Process all JARs.
*/
private void processJars() {
log("Starting scan.", verboseLevel);
long start = System.currentTimeMillis();
@SuppressWarnings("unchecked")
Iterator<Resource> iter = (Iterator<Resource>) jarResources.iterator();
int checked = 0;
int errors = 0;
while (iter.hasNext()) {
final Resource r = iter.next();
if (!r.isExists()) {
throw new BuildException("JAR resource does not exist: " + r.getName());
}
if (!(r instanceof FileResource)) {
throw new BuildException("Only filesystem resource are supported: " + r.getName()
+ ", was: " + r.getClass().getName());
}
File jarFile = ((FileResource) r).getFile();
if (!checkJarFile(jarFile)) {
errors++;
}
checked++;
}
log(String.format(Locale.ENGLISH,
"Scanned %d JAR file(s) for licenses (in %.2fs.), %d error(s).",
checked, (System.currentTimeMillis() - start) / 1000.0, errors),
errors > 0 ? Project.MSG_ERR : Project.MSG_INFO);
}
/**
* Check a single JAR file.
*/
private boolean checkJarFile(File jarFile) {
log("Scanning: " + jarFile.getPath(), verboseLevel);
// Get the expected license path base from the mapper and search for license files.
Map<File, LicenseType> foundLicenses = new LinkedHashMap<File, LicenseType>();
List<File> expectedLocations = new ArrayList<File>();
outer:
for (String mappedPath : licenseMapper.mapFileName(jarFile.getPath())) {
for (LicenseType licenseType : LicenseType.values()) {
File licensePath = new File(mappedPath + licenseType.licenseFileSuffix());
if (licensePath.exists()) {
foundLicenses.put(licensePath, licenseType);
log(" FOUND " + licenseType.name() + " license at " + licensePath.getPath(),
verboseLevel);
// We could continue scanning here to detect duplicate associations?
break outer;
} else {
expectedLocations.add(licensePath);
}
}
}
// Check for NOTICE files.
for (Map.Entry<File, LicenseType> e : foundLicenses.entrySet()) {
LicenseType license = e.getValue();
String licensePath = e.getKey().getAbsolutePath();
String baseName = licensePath.substring(
0, licensePath.length() - license.licenseFileSuffix().length());
File noticeFile = new File(baseName + license.noticeFileSuffix());
if (noticeFile.exists()) {
log(" FOUND NOTICE file at " + noticeFile.getAbsolutePath(), verboseLevel);
} else {
if (license.isNoticeRequired()) {
this.failures = true;
log("MISSING NOTICE for the license file:\n "
+ licensePath + "\n Expected location below:\n "
+ noticeFile.getAbsolutePath(), Project.MSG_ERR);
}
}
}
// In case there is something missing, complain.
if (foundLicenses.isEmpty()) {
this.failures = true;
StringBuilder message = new StringBuilder();
message.append(
"MISSING LICENSE for the following file:\n " + jarFile.getAbsolutePath()
+ "\n Expected locations below:\n");
for (File location : expectedLocations) {
message.append(" => ").append(location.getAbsolutePath()).append("\n");
}
log(message.toString(), Project.MSG_ERR);
return false;
}
return true;
}
}

View File

@ -23,7 +23,6 @@ package org.apache.lucene.validation;
*
**/
public enum LicenseType {
ASL("Apache Software License 2.0", true),
BSD("Berkeley Software Distribution", true),
BSD_LIKE("BSD like license", true),//BSD like just means someone has taken the BSD license and put in their name, copyright, or it's a very similar license.
@ -35,14 +34,12 @@ public enum LicenseType {
PD("Public Domain", false),
//SUNBCLA("Sun Binary Code License Agreement"),
SUN("Sun Open Source License", false),
FAKE("FAKE license - not needed", false)
;
COMPOUND("Compound license (see NOTICE).", true),
FAKE("FAKE license - not needed", false);
private String display;
private boolean noticeRequired;
LicenseType(String display, boolean noticeRequired) {
this.display = display;
this.noticeRequired = noticeRequired;
@ -56,11 +53,24 @@ public enum LicenseType {
return display;
}
public String toString() {
return "LicenseType{" +
"display='" + display + '\'' +
'}';
}
/**
* Expected license file suffix for a given license type.
*/
public String licenseFileSuffix() {
return "-LICENSE-" + this.name() + ".txt";
}
/**
* Expected notice file suffix for a given license type.
*/
public String noticeFileSuffix() {
return "-NOTICE.txt";
}
}

View File

@ -7,6 +7,7 @@ licenced on the terms of (inter alia): GPL, LGPL, MPL or CC-SA licenses.
Part-of-speech tags were added in Morfologik project and are not found
in the data from sjp.pl.
-----
BSD-licensed dictionary of Polish (SGJP)
http://sgjp.pl/morfeusz/

View File

@ -1,8 +1,9 @@
This product includes BSD-licensed software developed by Dawid Weiss and Marcin Miłkowski
(http://morfologik.blogspot.com/).
This product includes data from Polish ispell/myspell dictionary (http://www.sjp.pl/slownik/en/)
licenced on the terms of (inter alia) LGPL and Creative Commons ShareAlike.
This JAR contains and makes use of data from Polish ispell/myspell
dictionaries hosted at http://www.sjp.pl/slownik/en/ and is
licenced on the terms of (inter alia): GPL, LGPL, MPL or CC-SA licenses.
This product includes data from BSD-licensed dictionary of Polish (SGJP)
(http://sgjp.pl/morfeusz/)

View File

@ -18,111 +18,47 @@
-->
<project name="modules" default="test" basedir=".">
<!-- TODO: at some point we should probably iterate like contrib-crawl -->
<target name="test" description="Test all modules">
<dirname file="${ant.file.modules}" property="modules.dir"/>
<import file="${modules.dir}/../lucene/tools/custom-tasks.xml" />
<macrodef name="forall-modules">
<attribute name="target" />
<sequential>
<subant target="test" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="facet" includes="build.xml" />
<fileset dir="queries" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
<fileset dir="join" includes="build.xml" />
<fileset dir="queryparser" includes="build.xml" />
<fileset dir="suggest" includes="build.xml" />
<subant target="@{target}" inheritall="false" failonerror="true">
<fileset dir="${modules.dir}" includes="*/build.xml" />
</subant>
</sequential>
</macrodef>
<target name="test" description="Test all modules">
<forall-modules target="test" />
</target>
<target name="compile" description="Compile all modules" depends="validate">
<sequential>
<subant target="compile" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="facet" includes="build.xml" />
<fileset dir="queries" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
<fileset dir="join" includes="build.xml" />
<fileset dir="queryparser" includes="build.xml" />
<fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
<forall-modules target="compile" />
</target>
<target name="compile-test" description="Compile all tests">
<sequential>
<subant target="compile-test" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="facet" includes="build.xml" />
<fileset dir="queries" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
<fileset dir="join" includes="build.xml" />
<fileset dir="queryparser" includes="build.xml" />
<fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
<forall-modules target="compile-test" />
</target>
<target name="javadocs" description="Generate javadocs">
<sequential>
<subant target="javadocs" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="facet" includes="build.xml" />
<fileset dir="queries" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
<fileset dir="join" includes="build.xml" />
<fileset dir="queryparser" includes="build.xml" />
<fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
<forall-modules target="javadocs" />
</target>
<target name="generate-maven-artifacts" description="Generate Maven Artifacts for Modules">
<sequential>
<ant target="get-maven-poms" dir=".."/>
<subant target="dist-maven" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="facet" includes="build.xml" />
<fileset dir="queries" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
<fileset dir="join" includes="build.xml" />
<fileset dir="queryparser" includes="build.xml" />
<fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
<ant target="get-maven-poms" dir=".."/>
<forall-modules target="dist-maven" />
</target>
<target name="validate">
<sequential>
<subant target="validate" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="facet" includes="build.xml" />
<fileset dir="queries" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
<fileset dir="join" includes="build.xml" />
<fileset dir="queryparser" includes="build.xml" />
<fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
<target name="validate" description="Validate legal stuff.">
<!-- The order here is important because forall compiles tools. -->
<forall-modules target="validate" />
<license-check-macro dir="${basedir}" />
</target>
<target name="clean" description="Clean all modules">
<sequential>
<delete dir="dist"/>
<subant target="clean" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="facet" includes="build.xml" />
<fileset dir="queries" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
<fileset dir="join" includes="build.xml" />
<fileset dir="queryparser" includes="build.xml" />
<fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
<delete dir="dist"/>
<forall-modules target="clean" />
</target>
</project>

View File

@ -130,7 +130,7 @@
<target name="compile" description="Compile the source code."
depends="compile-core, compile-contrib"/>
<target name="test" description="Validate, then run core, solrj, and contrib unit tests."
depends="validate-solr, test-jsp, test-core, test-contrib"/>
depends="validate, test-jsp, test-core, test-contrib"/>
<target name="test-core" description="Runs the core and solrj unit tests."
depends="test-solr-core, test-solrj"/>
<target name="compile-test" description="Compile unit tests."
@ -177,6 +177,23 @@
</ant>
</target>
<!-- Validation (license/ notice checks). -->
<target name="validate" depends="compile-tools" description="Validate legal stuff.">
<license-check-macro dir="${basedir}">
<additional-excludes>
<!-- Exclude start.jar only (it'd be weird to have a license file there?) -->
<exclude name="example/start.jar" />
<exclude name="example/exampledocs/post.jar" />
</additional-excludes>
<additional-filters>
<replaceregex pattern="/jetty-util([^/]+)$" replace="/jetty-util" flags="gi" />
<replaceregex pattern="/jetty-6([^/]+)$" replace="/jetty" flags="gi" />
<replaceregex pattern="/jsp-2.1-glassfish([^/]+)$" replace="/jsp-2.1-glassfish" flags="gi" />
<replaceregex pattern="/jsp-api-2.1-glassfish([^/]+)$" replace="/jsp-api-2.1-glassfish" flags="gi" />
</additional-filters>
</license-check-macro>
</target>
<!-- Clean targets -->
<target name="clean" description="Cleans compiled files and other temporary artifacts.">
<delete dir="build" />

View File

@ -122,48 +122,8 @@
</sequential>
</macrodef>
<target name="validate" depends="validate-solr"/>
<target name="validate-solr" depends="check-legal-solr" unless="validated-solr.uptodate"/>
<target name="check-legal-solr" depends="compile-tools">
<java classname="org.apache.lucene.validation.DependencyChecker" failonerror="true" fork="true">
<classpath>
<path refid="tools.runtime.classpath" />
</classpath>
<!-- TODO: it might be better to just automatically find all directories that contain jar files, but that could take a
long time. This should be faster, but we could miss a directory
-->
<!-- Solr -->
<arg value="-c" />
<arg value="${common-solr.dir}/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/contrib/analysis-extras/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/contrib/clustering/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/contrib/dataimporthandler/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/contrib/dataimporthandler-extras/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/contrib/extraction/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/contrib/uima/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/contrib/velocity/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/example/example-DIH/solr/db/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/example/example-DIH/solr/mail/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/example/example/lib" />
<arg value="-c" />
<arg value="${common-solr.dir}/core/src/test-files/solr/lib" />
</java>
<property name="validated-solr.uptodate" value="true"/>
<target name="validate" depends="compile-tools">
</target>
<path id="tools.runtime.classpath">
<pathelement location="${common.dir}/build/tools/classes/java"/>
</path>
<target name="init-dist" >
<mkdir dir="${build.dir}"/>

View File

@ -0,0 +1,3 @@
jsonic - simple json encoder/decoder for java
http://jsonic.sourceforge.jp/

View File

@ -0,0 +1,3 @@
langdetect
http://code.google.com/p/language-detection/

View File

@ -1,21 +0,0 @@
Copyright (c) 2004-2008 QOS.ch
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -1,25 +0,0 @@
=========================================================================
== SLF4J Notice -- http://www.slf4j.org/license.html ==
=========================================================================
Copyright (c) 2004-2008 QOS.ch
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.