Merged /lucene/dev/trunk:r1522766-1523395

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5207@1523396 13f79535-47bb-0310-9956-ffa450edef68

dev-tools/scripts/checkJavaDocs.py

@@ -80,6 +80,8 @@ def checkClassDetails(fullPath):
   Checks for invalid HTML in the full javadocs under each field/method.
   """
 
+  isAttributeSource = fullPath.endswith('AttributeSource.html')
+
   # TODO: only works with java7 generated javadocs now!
   with open(fullPath, encoding='UTF-8') as f:
     desc = None
@@ -87,6 +89,12 @@ def checkClassDetails(fullPath):
     item = None
     errors = []
     for line in f.readlines():
+
+      if isAttributeSource:
+        # Work around Javadocs bug that fails to escape the <T> type parameter in {@link #getAttribute} and {@link #addAttribute}
+        line = line.replace('<code>getAttribute(java.lang.Class<T>)</code>', '<code>getAttribute(java.lang.Class)</code>')
+        line = line.replace('<code>addAttribute(java.lang.Class<T>)</code>', '<code>addAttribute(java.lang.Class)</code>')
+      
       m = reH3.search(line)
       if m is not None:
         if desc is not None:

extra-targets.xml

@@ -79,11 +79,7 @@
     <local name="svn.checkprops.failed"/>
     <local name="svn.unversioned.failed"/>
     <local name="svn.changed.failed"/>
-    <script language="groovy" taskname="svn">
-      <classpath>
-        <path refid="groovy.classpath"/>
-        <path refid="svnkit.classpath"/>
-      </classpath><![CDATA[
+    <groovy taskname="svn" classpathref="svnkit.classpath"><![CDATA[
       import org.tmatesoft.svn.core.*;
       import org.tmatesoft.svn.core.wc.*;
       
@@ -99,7 +95,7 @@
       
       Set missingProps = new TreeSet(), unversioned = new TreeSet(), changed = new TreeSet();
 
-      self.log('Getting all versioned and unversioned files...');
+      task.log('Getting all versioned and unversioned files...');
       statusClient.doStatus(basedir, SVNRevision.WORKING, SVNDepth.fromRecurse(true), false, true, false, false, {
         status ->
         SVNStatusType nodeStatus = status.getNodeStatus();
@@ -114,12 +110,12 @@
         }
       } as ISVNStatusHandler, null);
 
-      self.log('Filtering files with existing svn:eol-style...');
+      task.log('Filtering files with existing svn:eol-style...');
       wcClient.doGetProperty(basedir, 'svn:eol-style', SVNRevision.WORKING, SVNRevision.WORKING, true, {
         file, prop -> missingProps.remove(convertRelative(file));
       } as ISVNPropertyHandler);
       
-      self.log('Filtering files with binary svn:mime-type...');
+      task.log('Filtering files with binary svn:mime-type...');
       wcClient.doGetProperty(basedir, 'svn:mime-type', SVNRevision.WORKING, SVNRevision.WORKING, true, {
         file, prop ->
         prop = SVNPropertyValue.getPropertyAsString(prop.getValue());
@@ -134,7 +130,7 @@
       project.setProperty('svn.checkprops.failed', convertSet2String(missingProps));
       project.setProperty('svn.unversioned.failed', convertSet2String(unversioned));
       project.setProperty('svn.changed.failed', convertSet2String(changed));
-    ]]></script>
+    ]]></groovy>
     <fail if="svn.checkprops.failed"
       message="The following files are missing svn:eol-style (or binary svn:mime-type):${line.separator}${svn.checkprops.failed}"/>
     <fail if="svn.unversioned.failed"

lucene/common-build.xml

@@ -2066,6 +2066,9 @@ ${tests-output}/junit4-*.suites     - per-JVM executed suites
   <target name="resolve-groovy" unless="groovy.loaded" depends="ivy-availability-check,ivy-configure">
     <ivy:cachepath organisation="org.codehaus.groovy" module="groovy-all" revision="2.1.5"
       inline="true" conf="default" type="jar" transitive="true" pathid="groovy.classpath"/>
+    <taskdef name="groovy"
+      classname="org.codehaus.groovy.ant.Groovy"
+      classpathref="groovy.classpath"/>
     <property name="groovy.loaded" value="true"/>
   </target>
   

lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java

@@ -327,7 +327,7 @@ public class PostingsHighlighter {
    * @throws IllegalArgumentException if <code>field</code> was indexed without 
    *         {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
    */
-  public Map<String,Object[]> highlightFieldsAsObjects(String fieldsIn[], Query query, IndexSearcher searcher, int[] docidsIn, int maxPassagesIn[]) throws IOException {
+  protected Map<String,Object[]> highlightFieldsAsObjects(String fieldsIn[], Query query, IndexSearcher searcher, int[] docidsIn, int maxPassagesIn[]) throws IOException {
     if (fieldsIn.length < 1) {
       throw new IllegalArgumentException("fieldsIn must not be empty");
     }

lucene/tools/custom-tasks.xml

@@ -39,11 +39,13 @@
     
     <!-- Some typical snapshot/minimalized JAR suffixes. -->
     <replaceregex pattern="-min$" replace="" flags="gi" />
-    <replaceregex pattern="SNAPSHOT$" replace="" flags="gi" />
+    <replaceregex pattern="SNAPSHOT" replace="" flags="gi" />
 
     <!-- Typical version patterns. -->
     <replaceregex pattern="\.rc[0-9]+" replace="" flags="gi" />
-    <replaceregex pattern="\-(r)?([0-9\-\_\.])+((b(eta)?)|((a(lpha)?))([0-9\-\.])*)?(\-tests)?$" replace="" flags="gi" />
+    <replaceregex pattern="\-(r)?([0-9\-\_\.])+(([a-zA-Z]+)([0-9\-\.])*)?" replace="" flags="gi" />
+    
+    <replaceregex pattern="tests$" replace="-tests" flags="gi" />
 
     <!-- git hashcode pattern: its always 40 chars right? -->
     <replaceregex pattern="\-[a-z0-9]{40,40}$" replace="" flags="gi" />
@@ -57,7 +59,7 @@
     <sequential>
       <!-- LICENSE and NOTICE verification macro. -->
       <echo>License check under: @{dir}</echo>
-      <licenses licenseDirectory="@{licensedir}">
+      <licenses licenseDirectory="@{licensedir}" skipSnapshotsChecksum="${skipSnapshotsChecksum}">
         <fileset dir="@{dir}">
           <include name="**/*.jar" />
           <!-- Speed up scanning a bit. -->

lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java

@@ -54,6 +54,8 @@ public class LicenseCheckTask extends Task {
   private static final int CHECKSUM_BUFFER_SIZE = 8 * 1024;
   private static final int CHECKSUM_BYTE_MASK = 0xFF;
 
+  private boolean skipSnapshotsChecksum;
+  
   /**
    * All JAR files to check.
    */
@@ -104,6 +106,10 @@ public class LicenseCheckTask extends Task {
     licenseDirectory = file;
   }
   
+  public void setSkipSnapshotsChecksum(boolean skipSnapshotsChecksum) {
+    this.skipSnapshotsChecksum = skipSnapshotsChecksum;
+  }
+
   /**
    * Execute the task.
    */
@@ -113,6 +119,7 @@ public class LicenseCheckTask extends Task {
       throw new BuildException("Expected an embedded <licenseMapper>.");
     }
 
+    if (skipSnapshotsChecksum) log("Skipping checksum for SNAPSHOT dependencies", Project.MSG_INFO);
     jarResources.setProject(getProject());
     processJars();
 
@@ -161,48 +168,53 @@ public class LicenseCheckTask extends Task {
   private boolean checkJarFile(File jarFile) {
     log("Scanning: " + jarFile.getPath(), verboseLevel);
     
-    // validate the jar matches against our expected hash
-    final File checksumFile = new File(licenseDirectory, 
-                                       jarFile.getName() + "." + CHECKSUM_TYPE);
-    if (! (checksumFile.exists() && checksumFile.canRead()) ) {
-      log("MISSING " +CHECKSUM_TYPE+ " checksum file for: " + jarFile.getPath(), Project.MSG_ERR);
-      this.failures = true;
-      return false;
-    } else {
-      final String expectedChecksum = readChecksumFile(checksumFile);
-      try {
-        final MessageDigest md = MessageDigest.getInstance(CHECKSUM_TYPE);
-        byte[] buf = new byte[CHECKSUM_BUFFER_SIZE];
+    if (!skipSnapshotsChecksum || !jarFile.getName().contains("-SNAPSHOT")) {
+      // validate the jar matches against our expected hash
+      final File checksumFile = new File(licenseDirectory, 
+                                         jarFile.getName() + "." + CHECKSUM_TYPE);
+      if (! (checksumFile.exists() && checksumFile.canRead()) ) {
+        log("MISSING " +CHECKSUM_TYPE+ " checksum file for: " + jarFile.getPath(), Project.MSG_ERR);
+        log("EXPECTED " +CHECKSUM_TYPE+ " checksum file : " + checksumFile.getPath(), Project.MSG_ERR);
+        this.failures = true;
+        return false;
+      } else {
+        final String expectedChecksum = readChecksumFile(checksumFile);
         try {
-          FileInputStream fis = new FileInputStream(jarFile);
+          final MessageDigest md = MessageDigest.getInstance(CHECKSUM_TYPE);
+          byte[] buf = new byte[CHECKSUM_BUFFER_SIZE];
           try {
-            DigestInputStream dis = new DigestInputStream(fis, md);
+            FileInputStream fis = new FileInputStream(jarFile);
             try {
-              while (dis.read(buf, 0, CHECKSUM_BUFFER_SIZE) != -1) {
-                // NOOP
+              DigestInputStream dis = new DigestInputStream(fis, md);
+              try {
+                while (dis.read(buf, 0, CHECKSUM_BUFFER_SIZE) != -1) {
+                  // NOOP
+                }
+              } finally {
+                dis.close();
               }
             } finally {
-              dis.close();
+              fis.close();
             }
-          } finally {
-            fis.close();
+          } catch (IOException ioe) {
+            throw new BuildException("IO error computing checksum of file: " + jarFile, ioe);
+          }
+          final byte[] checksumBytes = md.digest();
+          final String checksum = createChecksumString(checksumBytes);
+          if ( ! checksum.equals(expectedChecksum) ) {
+            log("CHECKSUM FAILED for " + jarFile.getPath() + 
+                " (expected: \"" + expectedChecksum + "\" was: \"" + checksum + "\")", 
+                Project.MSG_ERR);
+            this.failures = true;
+            return false;
           }
-        } catch (IOException ioe) {
-          throw new BuildException("IO error computing checksum of file: " + jarFile, ioe);
-        }
-        final byte[] checksumBytes = md.digest();
-        final String checksum = createChecksumString(checksumBytes);
-        if ( ! checksum.equals(expectedChecksum) ) {
-          log("CHECKSUM FAILED for " + jarFile.getPath() + 
-              " (expected: \"" + expectedChecksum + "\" was: \"" + checksum + "\")", 
-              Project.MSG_ERR);
-          this.failures = true;
-          return false;
-        }
   
-      } catch (NoSuchAlgorithmException ae) {
-        throw new BuildException("Digest type " + CHECKSUM_TYPE + " not supported by your JVM", ae);
+        } catch (NoSuchAlgorithmException ae) {
+          throw new BuildException("Digest type " + CHECKSUM_TYPE + " not supported by your JVM", ae);
+        }
       }
+    } else if (skipSnapshotsChecksum) {
+      log("Skipping jar because it is a SNAPSHOT : " + jarFile.getAbsolutePath(), Project.MSG_INFO);
     }
     
     // Get the expected license path base from the mapper and search for license files.

solr/common-build.xml

@@ -283,7 +283,7 @@
   </macrodef>
 
   <target name="define-lucene-javadoc-url" depends="resolve-groovy" unless="lucene.javadoc.url">
-    <script language="groovy" classpathref="groovy.classpath"><![CDATA[
+    <groovy><![CDATA[
       String url, version = project.getProperty('version');
       if (version.contains('-SNAPSHOT')) {
         url = new File(project.getProperty('common.dir'), 'build' + File.separator + 'docs').toURI().toASCIIString();
@@ -292,9 +292,9 @@
         version = version.replace('.', '_');
         url = 'http://lucene.apache.org/core/' + version + '/';
       }
-      self.log('Using the following URL to refer to Lucene Javadocs: ' + url);
+      task.log('Using the following URL to refer to Lucene Javadocs: ' + url);
       project.setProperty('lucene.javadoc.url', url);
-    ]]></script>
+    ]]></groovy>
   </target>
 
   <target name="jar-src">

solr/example/etc/jetty.xml

@@ -51,7 +51,6 @@
     <Call name="addConnector">
       <Arg>
           <New class="org.eclipse.jetty.server.bio.SocketConnector">
-            <Call class="java.lang.System" name="setProperty"> <Arg>log4j.configuration</Arg> <Arg>etc/log4j.properties</Arg> </Call>
             <Set name="host"><SystemProperty name="jetty.host" /></Set>
             <Set name="port"><SystemProperty name="jetty.port" default="8983"/></Set>
             <Set name="maxIdleTime">50000</Set>

solr/example/etc/logging.properties

@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# To use this log config, start solr with the following system property: 
-# -Djava.util.logging.config.file=etc/logging.properties
-
-## Default global logging level:
-.level = INFO
-
-## Log every update command (add, delete, commit, ...)
-#org.apache.solr.update.processor.LogUpdateProcessor.level = FINE
-
-## Where to log (space separated list).
-handlers = java.util.logging.FileHandler
-
-java.util.logging.FileHandler.level = FINE
-
-java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter
-
-# 1 GB limit per file
-java.util.logging.FileHandler.limit = 1073741824
-
-# Log to the logs directory, with log files named solrxxx.log
-java.util.logging.FileHandler.pattern = ./logs/solr%u.log