HADOOP-6658. Exclude Private elements from generated Javadoc.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@937052 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -215,6 +215,8 @@ Trunk (unreleased changes)
     HADOOP-6705. Fix to work with 1.5 version of jiracli
     (Giridharan Kesavan)
 
+    HADOOP-6658. Exclude Private elements from generated Javadoc. (tomwhite)
+
   OPTIMIZATIONS
 
     HADOOP-6467. Improve the performance on HarFileSystem.listStatus(..).

build.xml

@@ -114,6 +114,8 @@
 
   <property name="jdiff.build.dir" value="${build.docs}/jdiff"/>
   <property name="jdiff.xml.dir" value="${lib.dir}/jdiff"/>
+  <property name="jdiff.stability" value="-unstable"/>
+  <property name="jdiff.compatibility" value=""/>
   <property name="jdiff.stable" value="0.20.2"/>
   <property name="jdiff.stable.javadoc" 
             value="http://hadoop.apache.org/core/docs/r${jdiff.stable}/api/"/>
@@ -867,7 +869,7 @@
     </uptodate>
   </target>
  
-  <target name="javadoc" description="Generate javadoc" depends="javadoc-uptodate"
+  <target name="javadoc" description="Generate javadoc" depends="jar, javadoc-uptodate"
        unless="javadoc.is.uptodate">
     <mkdir dir="${build.javadoc}"/>
     <javadoc
@@ -898,15 +900,19 @@
 
        <group title="Core" packages="org.apache.*"/>
        <group title="contrib: FailMon" packages="org.apache.hadoop.contrib.failmon*"/>
+       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet"
+               path="${build.dir}/${final.name}.jar"/>
     </javadoc>
   </target>	
 
   <target name="api-xml" depends="ivy-retrieve-jdiff,javadoc,write-null">
     <javadoc maxmemory="${javadoc.maxmemory}">
-       <doclet name="jdiff.JDiff"
+       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
-               path="${jdiff.jar}:${xerces.jar}">
+               path="${build.dir}/${final.name}.jar:${jdiff.jar}:${xerces.jar}">
          <param name="-apidir" value="${jdiff.xml.dir}"/>
          <param name="-apiname" value="hadoop-core ${version}"/>
+         <param name="${jdiff.stability}"/>
+         <param name="${jdiff.compatibility}"/>
        </doclet>
        <packageset dir="src/java"/>
        <classpath >
@@ -929,8 +935,8 @@
              destdir="${jdiff.build.dir}"
 	     sourceFiles="${jdiff.home}/Null.java"
 	     maxmemory="${javadoc.maxmemory}">
-       <doclet name="jdiff.JDiff"
+       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
-               path="${jdiff.jar}:${xerces.jar}">
+              path="${build.dir}/${final.name}.jar:${jdiff.jar}:${xerces.jar}">
          <param name="-oldapi" value="hadoop-core ${jdiff.stable}"/>
          <param name="-newapi" value="hadoop-core ${version}"/>
          <param name="-oldapidir" value="${jdiff.xml.dir}"/>
@@ -938,6 +944,8 @@
          <param name="-javadocold" value="${jdiff.stable.javadoc}"/>
          <param name="-javadocnew" value="../../api/"/>
          <param name="-stats"/>
+         <param name="${jdiff.stability}"/>
+         <param name="${jdiff.compatibility}"/>
        </doclet>
        <classpath >
          <path refid="classpath" />

ivy.xml

@@ -50,7 +50,7 @@
               extends="mandatory"/>
    <conf name="jetty" description="Jetty provides the in-VM HTTP daemon" extends="commons-logging"/>
 
-   <conf name="common" extends="runtime,mandatory,httpclient,ftp,jetty"
+   <conf name="common" extends="runtime,mandatory,httpclient,ftp,jetty,jdiff"
 		      description="common artifacts"/>
     <!--Testing pulls in everything-->
    <conf name="test" extends="master,common" description="the classpath needed to run tests"/>
@@ -91,11 +91,6 @@
       name="jdiff"
       rev="${jdiff.version}"
       conf="jdiff->default"/>
-    <dependency org="xerces"
-      name="xerces"
-      rev="${xerces.version}"
-      conf="jdiff->default">
-    </dependency>
 
     <dependency org="xmlenc"
       name="xmlenc"

src/java/org/apache/hadoop/classification/tools/ExcludePrivateAnnotationsJDiffDoclet.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.classification.tools;
+
+import com.sun.javadoc.DocErrorReporter;
+import com.sun.javadoc.LanguageVersion;
+import com.sun.javadoc.RootDoc;
+
+import jdiff.JDiff;
+
+/**
+ * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
+ * for excluding elements that are annotated with
+ * {@link org.apache.hadoop.classification.InterfaceAudience.Private} or
+ * {@link org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate}.
+ * It delegates to the JDiff Doclet, and takes the same options.
+ */
+public class ExcludePrivateAnnotationsJDiffDoclet {
+  
+  public static LanguageVersion languageVersion() {
+    return LanguageVersion.JAVA_1_5;
+  }
+  
+  public static boolean start(RootDoc root) {
+    System.out.println(
+	ExcludePrivateAnnotationsJDiffDoclet.class.getSimpleName());
+    return JDiff.start(RootDocProcessor.process(root));
+  }
+  
+  public static int optionLength(String option) {
+    Integer length = StabilityOptions.optionLength(option);
+    if (length != null) {
+      return length;
+    }
+    return JDiff.optionLength(option);
+  }
+  
+  public static boolean validOptions(String[][] options,
+      DocErrorReporter reporter) {
+    StabilityOptions.validOptions(options, reporter);
+    String[][] filteredOptions = StabilityOptions.filterOptions(options);
+    return JDiff.validOptions(filteredOptions, reporter);
+  }
+}

src/java/org/apache/hadoop/classification/tools/ExcludePrivateAnnotationsStandardDoclet.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.classification.tools;
+
+import com.sun.javadoc.DocErrorReporter;
+import com.sun.javadoc.LanguageVersion;
+import com.sun.javadoc.RootDoc;
+import com.sun.tools.doclets.standard.Standard;
+
+/**
+ * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
+ * for excluding elements that are annotated with
+ * {@link org.apache.hadoop.classification.InterfaceAudience.Private} or
+ * {@link org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate}.
+ * It delegates to the Standard Doclet, and takes the same options.
+ */
+public class ExcludePrivateAnnotationsStandardDoclet {
+  
+  public static LanguageVersion languageVersion() {
+    return LanguageVersion.JAVA_1_5;
+  }
+  
+  public static boolean start(RootDoc root) {
+    System.out.println(
+	ExcludePrivateAnnotationsStandardDoclet.class.getSimpleName());
+    return Standard.start(RootDocProcessor.process(root));
+  }
+  
+  public static int optionLength(String option) {
+    Integer length = StabilityOptions.optionLength(option);
+    if (length != null) {
+      return length;
+    }
+    return Standard.optionLength(option);
+  }
+  
+  public static boolean validOptions(String[][] options,
+      DocErrorReporter reporter) {
+    StabilityOptions.validOptions(options, reporter);
+    String[][] filteredOptions = StabilityOptions.filterOptions(options);
+    return Standard.validOptions(filteredOptions, reporter);
+  }
+}

src/java/org/apache/hadoop/classification/tools/RootDocProcessor.java

@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.classification.tools;
+
+import com.sun.javadoc.AnnotationDesc;
+import com.sun.javadoc.AnnotationTypeDoc;
+import com.sun.javadoc.ClassDoc;
+import com.sun.javadoc.ConstructorDoc;
+import com.sun.javadoc.Doc;
+import com.sun.javadoc.FieldDoc;
+import com.sun.javadoc.MethodDoc;
+import com.sun.javadoc.PackageDoc;
+import com.sun.javadoc.ProgramElementDoc;
+import com.sun.javadoc.RootDoc;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Process the {@link RootDoc} by substituting with (nested) proxy objects that
+ * exclude elements with Private or LimitedPrivate annotations.
+ * <p>
+ * Based on code from http://www.sixlegs.com/blog/java/exclude-javadoc-tag.html.
+ */
+class RootDocProcessor {
+  
+  static String stability = StabilityOptions.UNSTABLE_OPTION;
+  
+  public static RootDoc process(RootDoc root) {
+    return (RootDoc) process(root, RootDoc.class);
+  }
+  
+  private static Object process(Object obj, Class<?> type) { 
+    if (obj == null) { 
+      return null; 
+    } 
+    Class<?> cls = obj.getClass(); 
+    if (cls.getName().startsWith("com.sun.")) { 
+      return getProxy(obj); 
+    } else if (obj instanceof Object[]) { 
+      Class<?> componentType = type.isArray() ? type.getComponentType() 
+	  : cls.getComponentType();
+      Object[] array = (Object[]) obj;
+      Object[] newArray = (Object[]) Array.newInstance(componentType,
+	  array.length); 
+      for (int i = 0; i < array.length; ++i) {
+        newArray[i] = process(array[i], componentType);
+      }
+      return newArray;
+    } 
+    return obj; 
+  }
+  
+  private static Map<Object, Object> proxies =
+    new WeakHashMap<Object, Object>(); 
+  
+  private static Object getProxy(Object obj) { 
+    Object proxy = proxies.get(obj); 
+    if (proxy == null) { 
+      proxy = Proxy.newProxyInstance(obj.getClass().getClassLoader(), 
+        obj.getClass().getInterfaces(), new ExcludeHandler(obj)); 
+      proxies.put(obj, proxy); 
+    } 
+    return proxy; 
+  } 
+
+  private static class ExcludeHandler implements InvocationHandler {
+    private Object target;
+
+    public ExcludeHandler(Object target) {
+      this.target = target;
+    }
+    
+    public Object invoke(Object proxy, Method method, Object[] args)
+	throws Throwable {
+      String methodName = method.getName();
+      if (target instanceof Doc) {
+	if (methodName.equals("isIncluded")) {
+	  Doc doc = (Doc) target;
+	  return !exclude(doc) && doc.isIncluded();
+	}
+	if (target instanceof RootDoc) {
+	  if (methodName.equals("classes")) {
+	    return filter(((RootDoc) target).classes(), ClassDoc.class);
+	  } else if (methodName.equals("specifiedClasses")) {
+	    return filter(((RootDoc) target).specifiedClasses(), ClassDoc.class);
+	  } else if (methodName.equals("specifiedPackages")) {
+	    return filter(((RootDoc) target).specifiedPackages(), PackageDoc.class);
+	  }
+	} else if (target instanceof ClassDoc) {
+	  if (isFiltered(args)) {
+	    if (methodName.equals("methods")) {
+	      return filter(((ClassDoc) target).methods(true), MethodDoc.class);
+	    } else if (methodName.equals("fields")) {
+	      return filter(((ClassDoc) target).fields(true), FieldDoc.class);
+	    } else if (methodName.equals("innerClasses")) {
+	      return filter(((ClassDoc) target).innerClasses(true),
+		  ClassDoc.class);
+	    } else if (methodName.equals("constructors")) {
+	      return filter(((ClassDoc) target).constructors(true),
+		  ConstructorDoc.class);
+	    }
+	  }
+	} else if (target instanceof PackageDoc) {
+	  if (methodName.equals("allClasses")) {
+	    if (isFiltered(args)) {
+	      return filter(((PackageDoc) target).allClasses(true),
+		ClassDoc.class);
+	    } else {
+	      return filter(((PackageDoc) target).allClasses(), ClassDoc.class);  
+	    }
+	  } else if (methodName.equals("annotationTypes")) {
+	    return filter(((PackageDoc) target).annotationTypes(),
+		AnnotationTypeDoc.class);
+	  } else if (methodName.equals("enums")) {
+	    return filter(((PackageDoc) target).enums(),
+		ClassDoc.class);
+	  } else if (methodName.equals("errors")) {
+	    return filter(((PackageDoc) target).errors(),
+		ClassDoc.class);
+	  } else if (methodName.equals("exceptions")) {
+	    return filter(((PackageDoc) target).exceptions(),
+		ClassDoc.class);
+	  } else if (methodName.equals("interfaces")) {
+	    return filter(((PackageDoc) target).interfaces(),
+		ClassDoc.class);
+	  } else if (methodName.equals("ordinaryClasses")) {
+	    return filter(((PackageDoc) target).ordinaryClasses(),
+		ClassDoc.class);
+	  }
+	}
+      }
+
+      if (args != null) {
+	if (methodName.equals("compareTo") || methodName.equals("equals")
+	    || methodName.equals("overrides")
+	    || methodName.equals("subclassOf")) {
+	  args[0] = unwrap(args[0]);
+	}
+      }
+      try {
+	return process(method.invoke(target, args), method.getReturnType());
+      } catch (InvocationTargetException e) {
+	throw e.getTargetException();
+      }
+    }
+      
+    private static boolean exclude(Doc doc) {
+      AnnotationDesc[] annotations = null;
+      if (doc instanceof ProgramElementDoc) {
+	annotations = ((ProgramElementDoc) doc).annotations();
+      } else if (doc instanceof PackageDoc) {
+	annotations = ((PackageDoc) doc).annotations();
+      }
+      if (annotations != null) {
+	for (AnnotationDesc annotation : annotations) {
+	  String qualifiedTypeName = annotation.annotationType().qualifiedTypeName();
+	  if (qualifiedTypeName.equals(
+	        InterfaceAudience.Private.class.getCanonicalName())
+	    || qualifiedTypeName.equals(
+                InterfaceAudience.LimitedPrivate.class.getCanonicalName())) {
+	    return true;
+	  }
+	  if (stability.equals(StabilityOptions.EVOLVING_OPTION)) {
+	    if (qualifiedTypeName.equals(
+		InterfaceStability.Unstable.class.getCanonicalName())) {
+	      return true;
+	    }
+	  }
+	  if (stability.equals(StabilityOptions.STABLE_OPTION)) {
+	    if (qualifiedTypeName.equals(
+		InterfaceStability.Unstable.class.getCanonicalName())
+              || qualifiedTypeName.equals(
+  		InterfaceStability.Evolving.class.getCanonicalName())) {
+	      return true;
+	    }
+	  }
+	}
+      }
+      return false;
+    }
+      
+    private static Object[] filter(Doc[] array, Class<?> componentType) {
+      if (array == null || array.length == 0) {
+	return array;
+      }
+      List<Object> list = new ArrayList<Object>(array.length);
+      for (Doc entry : array) {
+	if (!exclude(entry)) {
+	  list.add(process(entry, componentType));
+	}
+      }
+      return list.toArray((Object[]) Array.newInstance(componentType, list
+	  .size()));
+    }
+
+    private Object unwrap(Object proxy) {
+      if (proxy instanceof Proxy)
+	return ((ExcludeHandler) Proxy.getInvocationHandler(proxy)).target;
+      return proxy;
+    }
+      
+    private boolean isFiltered(Object[] args) {
+      return args != null && Boolean.TRUE.equals(args[0]);
+    }
+
+  }
+
+}

src/java/org/apache/hadoop/classification/tools/StabilityOptions.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.classification.tools;
+
+import com.sun.javadoc.DocErrorReporter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+class StabilityOptions {
+  public static final String STABLE_OPTION = "-stable";
+  public static final String EVOLVING_OPTION = "-evolving";
+  public static final String UNSTABLE_OPTION = "-unstable";
+
+  public static Integer optionLength(String option) {
+    String opt = option.toLowerCase();
+    if (opt.equals(UNSTABLE_OPTION)) return 1;
+    if (opt.equals(EVOLVING_OPTION)) return 1;
+    if (opt.equals(STABLE_OPTION)) return 1;
+    return null;
+  }
+
+  public static void validOptions(String[][] options,
+      DocErrorReporter reporter) {
+    for (int i = 0; i < options.length; i++) {
+      String opt = options[i][0].toLowerCase();
+      if (opt.equals(UNSTABLE_OPTION)) {
+	RootDocProcessor.stability = UNSTABLE_OPTION;
+      } else if (opt.equals(EVOLVING_OPTION)) {
+	RootDocProcessor.stability = EVOLVING_OPTION;
+      } else if (opt.equals(STABLE_OPTION)) {
+	RootDocProcessor.stability = STABLE_OPTION;	
+      }
+    }
+  }
+  
+  public static String[][] filterOptions(String[][] options) {
+    List<String[]> optionsList = new ArrayList<String[]>();
+    for (int i = 0; i < options.length; i++) {
+      if (!options[i][0].equalsIgnoreCase(UNSTABLE_OPTION)
+	  && !options[i][0].equalsIgnoreCase(EVOLVING_OPTION)
+	  && !options[i][0].equalsIgnoreCase(STABLE_OPTION)) {
+	optionsList.add(options[i]);
+      }
+    }
+    String[][] filteredOptions = new String[optionsList.size()][];
+    int i = 0;
+    for (String[] option : optionsList) {
+      filteredOptions[i++] = option;
+    }
+    return filteredOptions;
+  }
+
+}

src/java/org/apache/hadoop/classification/tools/package-info.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.LimitedPrivate({COMMON, AVRO, CHUKWA, HBASE, HDFS, HIVE,
+  MAPREDUCE, PIG, ZOOKEEPER})
+package org.apache.hadoop.classification.tools;
+
+import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.*;
+
+import org.apache.hadoop.classification.InterfaceAudience;

src/test/aop/build/aop.xml

@@ -48,7 +48,7 @@
       encoding="${build.encoding}" 
       srcdir="${java.src.dir};${build.src};${test.src.dir}/aop" 
       includes="org/apache/hadoop/**/*.java, org/apache/hadoop/**/*.aj"
-      excludes="org/apache/hadoop/record/**/*"
+      excludes="org/apache/hadoop/classification/tools/**/*, org/apache/hadoop/record/**/*"
       destDir="${build.classes}"
       debug="${javac.debug}"
       target="${javac.version}"