From 2d35d0063aabca0ce2bbe69f91ccfd8b912f090c Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 14 Feb 2013 01:30:50 +0000 Subject: [PATCH] HADOOP-8924. Add maven plugin alternative to shell script to save package-info.java. Contributed by Alejandro Abdelnur and Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1446018 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop-common/dev-support/saveVersion.sh | 67 ---- hadoop-common-project/hadoop-common/pom.xml | 70 +++- .../hadoop/HadoopVersionAnnotation.java | 74 ---- .../org/apache/hadoop/util/VersionInfo.java | 90 +++-- .../resources/common-version-info.properties | 25 ++ hadoop-maven-plugins/pom.xml | 76 ++++ .../apache/hadoop/maven/plugin/util/Exec.java | 118 ++++++ .../maven/plugin/util/FileSetUtils.java | 61 ++++ .../plugin/versioninfo/VersionInfoMojo.java | 343 ++++++++++++++++++ hadoop-project/pom.xml | 5 + .../hadoop-yarn/hadoop-yarn-common/pom.xml | 58 ++- .../hadoop-yarn-common/scripts/saveVersion.sh | 62 ---- .../hadoop/yarn/util/YarnVersionInfo.java | 42 +-- .../resources/yarn-version-info.properties | 25 ++ pom.xml | 1 + 16 files changed, 831 insertions(+), 289 deletions(-) delete mode 100755 hadoop-common-project/hadoop-common/dev-support/saveVersion.sh delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java create mode 100644 hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties create mode 100644 hadoop-maven-plugins/pom.xml create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java delete mode 100755 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/scripts/saveVersion.sh create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index db55a9eae9d..f0e230a5b20 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -13,6 +13,9 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9253. Capture ulimit info in the logs at service start time. (Arpit Gupta via suresh) + HADOOP-8924. Add maven plugin alternative to shell script to save + package-info.java. (Chris Nauroth via suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/dev-support/saveVersion.sh b/hadoop-common-project/hadoop-common/dev-support/saveVersion.sh deleted file mode 100755 index d11a4cf75c2..00000000000 --- a/hadoop-common-project/hadoop-common/dev-support/saveVersion.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This file is used to generate the package-info.java class that -# records the version, revision, branch, user, timestamp, and url -unset LANG -unset LC_CTYPE -unset LC_TIME -version=$1 -build_dir=$2 -user=`whoami | tr '\n\r' '\n'` -date=`date` -cwd=`pwd` -if git rev-parse HEAD 2>/dev/null > /dev/null ; then - revision=`git log -1 --pretty=format:"%H"` - hostname=`hostname` - branch=`git branch | sed -n -e 's/^* //p'` - url="git://${hostname}${cwd}" -elif [ -d .svn ]; then - revision=`svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'` - url=`svn info | sed -n -e 's/^URL: \(.*\)/\1/p'` - # Get canonical branch (branches/X, tags/X, or trunk) - branch=`echo $url | sed -n -e 's,.*\(branches/.*\)$,\1,p' \ - -e 's,.*\(tags/.*\)$,\1,p' \ - -e 's,.*trunk$,trunk,p'` -else - revision="Unknown" - branch="Unknown" - url="file://$cwd" -fi - -which md5sum > /dev/null -if [ "$?" = "0" ] ; then - srcChecksum=`find src/main/java -name '*.java' | LC_ALL=C sort | xargs md5sum | md5sum | cut -d ' ' -f 1` -else - srcChecksum="Not Available" -fi - -mkdir -p $build_dir/org/apache/hadoop -cat << EOF | \ - sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \ - -e "s|URL|$url|" -e "s/REV/$revision/" \ - -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \ - > $build_dir/org/apache/hadoop/package-info.java -/* - * Generated by src/saveVersion.sh - */ -@HadoopVersionAnnotation(version="VERSION", revision="REV", branch="BRANCH", - user="USER", date="DATE", url="URL", - srcChecksum="SRCCHECKSUM") -package org.apache.hadoop; -EOF diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 9545c66eb63..f5f2e9c3f9b 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -249,7 +249,61 @@ + + + + ${basedir}/src/main/resources + + common-version-info.properties + + false + + + ${basedir}/src/main/resources + + common-version-info.properties + + true + + + + org.apache.hadoop + hadoop-maven-plugins + + + version-info + + version-info + + + + ${basedir}/src/main + + java/**/*.java + proto/**/*.proto + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${startKdc} + ${kdc.resource.dir} + + + org.apache.avro avro-maven-plugin @@ -283,22 +337,6 @@ - - save-version - generate-sources - - run - - - - - - - - - - generate-test-sources generate-test-sources diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java deleted file mode 100644 index 132210f1a9f..00000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop; - -import java.lang.annotation.*; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * A package attribute that captures the version of Hadoop that was compiled. - */ -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.PACKAGE) -@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) -@InterfaceStability.Unstable -public @interface HadoopVersionAnnotation { - - /** - * Get the Hadoop version - * @return the version string "0.6.3-dev" - */ - String version(); - - /** - * Get the username that compiled Hadoop. - */ - String user(); - - /** - * Get the date when Hadoop was compiled. - * @return the date in unix 'date' format - */ - String date(); - - /** - * Get the url for the subversion repository. - */ - String url(); - - /** - * Get the subversion revision. - * @return the revision number as a string (eg. "451451") - */ - String revision(); - - /** - * Get the branch from which this was compiled. - * @return The branch name, e.g. "trunk" or "branches/branch-0.20" - */ - String branch(); - - /** - * Get a checksum of the source files from which - * Hadoop was compiled. - * @return a string that uniquely identifies the source - **/ - String srcChecksum(); -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java index 7bde3ade14c..f2415590b0d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java @@ -20,41 +20,78 @@ package org.apache.hadoop.util; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.HadoopVersionAnnotation; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + /** - * This class finds the package info for Hadoop and the HadoopVersionAnnotation - * information. + * This class returns build information about Hadoop components. */ @InterfaceAudience.Private @InterfaceStability.Unstable public class VersionInfo { private static final Log LOG = LogFactory.getLog(VersionInfo.class); - private static Package myPackage; - private static HadoopVersionAnnotation version; - - static { - myPackage = HadoopVersionAnnotation.class.getPackage(); - version = myPackage.getAnnotation(HadoopVersionAnnotation.class); + private Properties info; + + protected VersionInfo(String component) { + info = new Properties(); + String versionInfoFile = component + "-version-info.properties"; + try { + InputStream is = Thread.currentThread().getContextClassLoader() + .getResourceAsStream(versionInfoFile); + info.load(is); + } catch (IOException ex) { + LogFactory.getLog(getClass()).warn("Could not read '" + + versionInfoFile + "', " + ex.toString(), ex); + } } - /** - * Get the meta-data for the Hadoop package. - * @return - */ - static Package getPackage() { - return myPackage; + protected String _getVersion() { + return info.getProperty("version", "Unknown"); } - + + protected String _getRevision() { + return info.getProperty("revision", "Unknown"); + } + + protected String _getBranch() { + return info.getProperty("branch", "Unknown"); + } + + protected String _getDate() { + return info.getProperty("date", "Unknown"); + } + + protected String _getUser() { + return info.getProperty("user", "Unknown"); + } + + protected String _getUrl() { + return info.getProperty("url", "Unknown"); + } + + protected String _getSrcChecksum() { + return info.getProperty("srcChecksum", "Unknown"); + } + + protected String _getBuildVersion(){ + return getVersion() + + " from " + _getRevision() + + " by " + _getUser() + + " source checksum " + _getSrcChecksum(); + } + + private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common"); /** * Get the Hadoop version. * @return the Hadoop version string, eg. "0.6.3-dev" */ public static String getVersion() { - return version != null ? version.version() : "Unknown"; + return COMMON_VERSION_INFO._getVersion(); } /** @@ -62,7 +99,7 @@ public class VersionInfo { * @return the revision number, eg. "451451" */ public static String getRevision() { - return version != null ? version.revision() : "Unknown"; + return COMMON_VERSION_INFO._getRevision(); } /** @@ -70,7 +107,7 @@ public class VersionInfo { * @return The branch name, e.g. "trunk" or "branches/branch-0.20" */ public static String getBranch() { - return version != null ? version.branch() : "Unknown"; + return COMMON_VERSION_INFO._getBranch(); } /** @@ -78,7 +115,7 @@ public class VersionInfo { * @return the compilation date in unix date format */ public static String getDate() { - return version != null ? version.date() : "Unknown"; + return COMMON_VERSION_INFO._getDate(); } /** @@ -86,14 +123,14 @@ public class VersionInfo { * @return the username of the user */ public static String getUser() { - return version != null ? version.user() : "Unknown"; + return COMMON_VERSION_INFO._getUser(); } /** * Get the subversion URL for the root Hadoop directory. */ public static String getUrl() { - return version != null ? version.url() : "Unknown"; + return COMMON_VERSION_INFO._getUrl(); } /** @@ -101,7 +138,7 @@ public class VersionInfo { * built. **/ public static String getSrcChecksum() { - return version != null ? version.srcChecksum() : "Unknown"; + return COMMON_VERSION_INFO._getSrcChecksum(); } /** @@ -109,14 +146,11 @@ public class VersionInfo { * revision, user and date. */ public static String getBuildVersion(){ - return VersionInfo.getVersion() + - " from " + VersionInfo.getRevision() + - " by " + VersionInfo.getUser() + - " source checksum " + VersionInfo.getSrcChecksum(); + return COMMON_VERSION_INFO._getBuildVersion(); } public static void main(String[] args) { - LOG.debug("version: "+ version); + LOG.debug("version: "+ getVersion()); System.out.println("Hadoop " + getVersion()); System.out.println("Subversion " + getUrl() + " -r " + getRevision()); System.out.println("Compiled by " + getUser() + " on " + getDate()); diff --git a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties new file mode 100644 index 00000000000..9a8575c6dea --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version=${pom.version} +revision=${version-info.scm.commit} +branch=${version-info.scm.branch} +user=${user.name} +date=${version-info.build.time} +url=${version-info.scm.uri} +srcChecksum=${version-info.source.md5} diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml new file mode 100644 index 00000000000..2c92c7d1930 --- /dev/null +++ b/hadoop-maven-plugins/pom.xml @@ -0,0 +1,76 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 2.0.4-SNAPSHOT + ../hadoop-project + + org.apache.hadoop + hadoop-maven-plugins + maven-plugin + Apache Hadoop Maven Plugins + + 3.0 + + + + org.apache.maven + maven-plugin-api + ${maven.dependency.version} + + + org.apache.maven + maven-core + ${maven.dependency.version} + + + org.apache.maven.plugin-tools + maven-plugin-annotations + ${maven.dependency.version} + provided + + + junit + junit + 3.8.1 + test + + + + + + org.apache.maven.plugins + maven-plugin-plugin + ${maven.dependency.version} + + true + + + + mojo-descriptor + + descriptor + + + + + + + diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java new file mode 100644 index 00000000000..144ee135623 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java @@ -0,0 +1,118 @@ +/* + * Copyright 2012 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.util; + +import org.apache.maven.plugin.Mojo; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * Exec is a helper class for executing an external process from a mojo. + */ +public class Exec { + private Mojo mojo; + + /** + * Creates a new Exec instance for executing an external process from the given + * mojo. + * + * @param mojo Mojo executing external process + */ + public Exec(Mojo mojo) { + this.mojo = mojo; + } + + /** + * Runs the specified command and saves each line of the command's output to + * the given list. + * + * @param command List containing command and all arguments + * @param output List in/out parameter to receive command output + * @return int exit code of command + */ + public int run(List command, List output) { + int retCode = 1; + ProcessBuilder pb = new ProcessBuilder(command); + try { + Process p = pb.start(); + OutputBufferThread stdOut = new OutputBufferThread(p.getInputStream()); + OutputBufferThread stdErr = new OutputBufferThread(p.getErrorStream()); + stdOut.start(); + stdErr.start(); + retCode = p.waitFor(); + if (retCode != 0) { + mojo.getLog().warn(command + " failed with error code " + retCode); + for (String s : stdErr.getOutput()) { + mojo.getLog().debug(s); + } + } else { + stdOut.join(); + stdErr.join(); + output.addAll(stdOut.getOutput()); + } + } catch (Exception ex) { + mojo.getLog().warn(command + " failed: " + ex.toString()); + } + return retCode; + } + + /** + * OutputBufferThread is a background thread for consuming and storing output + * of the external process. + */ + private static class OutputBufferThread extends Thread { + private List output; + private BufferedReader reader; + + /** + * Creates a new OutputBufferThread to consume the given InputStream. + * + * @param is InputStream to consume + */ + public OutputBufferThread(InputStream is) { + this.setDaemon(true); + output = new ArrayList(); + reader = new BufferedReader(new InputStreamReader(is)); + } + + @Override + public void run() { + try { + String line = reader.readLine(); + while (line != null) { + output.add(line); + line = reader.readLine(); + } + } catch (IOException ex) { + throw new RuntimeException("make failed with error code " + ex.toString()); + } + } + + /** + * Returns every line consumed from the input. + * + * @return List every line consumed from the input + */ + public List getOutput() { + return output; + } + } +} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java new file mode 100644 index 00000000000..73f2ca6ddbd --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java @@ -0,0 +1,61 @@ +/* + * Copyright 2012 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.util; + +import org.apache.maven.model.FileSet; +import org.codehaus.plexus.util.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * FileSetUtils contains helper methods for mojo implementations that need to + * work with a Maven FileSet. + */ +public class FileSetUtils { + + /** + * Returns a string containing every element of the given list, with each + * element separated by a comma. + * + * @param list List of all elements + * @return String containing every element, comma-separated + */ + private static String getCommaSeparatedList(List list) { + StringBuilder buffer = new StringBuilder(); + String separator = ""; + for (Object e : list) { + buffer.append(separator).append(e); + separator = ","; + } + return buffer.toString(); + } + + /** + * Converts a Maven FileSet to a list of File objects. + * + * @param source FileSet to convert + * @return List containing every element of the FileSet as a File + * @throws IOException if an I/O error occurs while trying to find the files + */ + @SuppressWarnings("unchecked") + public static List convertFileSetToFiles(FileSet source) throws IOException { + String includes = getCommaSeparatedList(source.getIncludes()); + String excludes = getCommaSeparatedList(source.getExcludes()); + return FileUtils.getFiles(new File(source.getDirectory()), includes, excludes); + } +} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java new file mode 100644 index 00000000000..b489c0a7c0d --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java @@ -0,0 +1,343 @@ +/* + * Copyright 2012 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.versioninfo; + +import org.apache.hadoop.maven.plugin.util.Exec; +import org.apache.hadoop.maven.plugin.util.FileSetUtils; +import org.apache.maven.model.FileSet; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +/** + * VersionInfoMojo calculates information about the current version of the + * codebase and exports the information as properties for further use in a Maven + * build. The version information includes build time, SCM URI, SCM branch, SCM + * commit, and an MD5 checksum of the contents of the files in the codebase. + */ +@Mojo(name="version-info", defaultPhase=LifecyclePhase.INITIALIZE) +public class VersionInfoMojo extends AbstractMojo { + + @Parameter(defaultValue="${project}") + private MavenProject project; + + @Parameter(required=true) + private FileSet source; + + @Parameter(defaultValue="version-info.build.time") + private String buildTimeProperty; + + @Parameter(defaultValue="version-info.source.md5") + private String md5Property; + + @Parameter(defaultValue="version-info.scm.uri") + private String scmUriProperty; + + @Parameter(defaultValue="version-info.scm.branch") + private String scmBranchProperty; + + @Parameter(defaultValue="version-info.scm.commit") + private String scmCommitProperty; + + @Parameter(defaultValue="git") + private String gitCommand; + + @Parameter(defaultValue="svn") + private String svnCommand; + + private enum SCM {NONE, SVN, GIT} + + @Override + public void execute() throws MojoExecutionException { + try { + SCM scm = determineSCM(); + project.getProperties().setProperty(buildTimeProperty, getBuildTime()); + project.getProperties().setProperty(scmUriProperty, getSCMUri(scm)); + project.getProperties().setProperty(scmBranchProperty, getSCMBranch(scm)); + project.getProperties().setProperty(scmCommitProperty, getSCMCommit(scm)); + project.getProperties().setProperty(md5Property, computeMD5()); + } catch (Throwable ex) { + throw new MojoExecutionException(ex.toString(), ex); + } + } + + /** + * Returns a string representing current build time. + * + * @return String representing current build time + */ + private String getBuildTime() { + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + return dateFormat.format(new Date()); + } + private List scmOut; + + /** + * Determines which SCM is in use (Subversion, git, or none) and captures + * output of the SCM command for later parsing. + * + * @return SCM in use for this build + * @throws Exception if any error occurs attempting to determine SCM + */ + private SCM determineSCM() throws Exception { + Exec exec = new Exec(this); + SCM scm = SCM.NONE; + scmOut = new ArrayList(); + int ret = exec.run(Arrays.asList(svnCommand, "info"), scmOut); + if (ret == 0) { + scm = SCM.SVN; + } else { + ret = exec.run(Arrays.asList(gitCommand, "branch"), scmOut); + if (ret == 0) { + ret = exec.run(Arrays.asList(gitCommand, "remote", "-v"), scmOut); + if (ret != 0) { + scm = SCM.NONE; + scmOut = null; + } else { + ret = exec.run(Arrays.asList(gitCommand, "log", "-n", "1"), scmOut); + if (ret != 0) { + scm = SCM.NONE; + scmOut = null; + } else { + scm = SCM.GIT; + } + } + } + } + if (scmOut != null) { + getLog().debug(scmOut.toString()); + } + getLog().info("SCM: " + scm); + return scm; + } + + /** + * Return URI and branch of Subversion repository. + * + * @param str String Subversion info output containing URI and branch + * @return String[] containing URI and branch + */ + private String[] getSvnUriInfo(String str) { + String[] res = new String[]{"Unknown", "Unknown"}; + try { + String path = str; + int index = path.indexOf("trunk"); + if (index > -1) { + res[0] = path.substring(0, index - 1); + res[1] = "trunk"; + } else { + index = path.indexOf("branches"); + if (index > -1) { + res[0] = path.substring(0, index - 1); + int branchIndex = index + "branches".length() + 1; + index = path.indexOf("/", branchIndex); + if (index > -1) { + res[1] = path.substring(branchIndex, index); + } else { + res[1] = path.substring(branchIndex); + } + } + } + } catch (Exception ex) { + getLog().warn("Could not determine URI & branch from SVN URI: " + str); + } + return res; + } + + /** + * Parses SCM output and returns URI of SCM. + * + * @param scm SCM in use for this build + * @return String URI of SCM + */ + private String getSCMUri(SCM scm) { + String uri = "Unknown"; + switch (scm) { + case SVN: + for (String s : scmOut) { + if (s.startsWith("URL:")) { + uri = s.substring(4).trim(); + uri = getSvnUriInfo(uri)[0]; + break; + } + } + break; + case GIT: + for (String s : scmOut) { + if (s.startsWith("origin") && s.endsWith("(fetch)")) { + uri = s.substring("origin".length()); + uri = uri.substring(0, uri.length() - "(fetch)".length()); + break; + } + } + break; + } + return uri.trim(); + } + + /** + * Parses SCM output and returns commit of SCM. + * + * @param scm SCM in use for this build + * @return String commit of SCM + */ + private String getSCMCommit(SCM scm) { + String commit = "Unknown"; + switch (scm) { + case SVN: + for (String s : scmOut) { + if (s.startsWith("Revision:")) { + commit = s.substring("Revision:".length()); + break; + } + } + break; + case GIT: + for (String s : scmOut) { + if (s.startsWith("commit")) { + commit = s.substring("commit".length()); + break; + } + } + break; + } + return commit.trim(); + } + + /** + * Parses SCM output and returns branch of SCM. + * + * @param scm SCM in use for this build + * @return String branch of SCM + */ + private String getSCMBranch(SCM scm) { + String branch = "Unknown"; + switch (scm) { + case SVN: + for (String s : scmOut) { + if (s.startsWith("URL:")) { + branch = s.substring(4).trim(); + branch = getSvnUriInfo(branch)[1]; + break; + } + } + break; + case GIT: + for (String s : scmOut) { + if (s.startsWith("*")) { + branch = s.substring("*".length()); + break; + } + } + break; + } + return branch.trim(); + } + + /** + * Reads and returns the full contents of the specified file. + * + * @param file File to read + * @return byte[] containing full contents of file + * @throws IOException if there is an I/O error while reading the file + */ + private byte[] readFile(File file) throws IOException { + RandomAccessFile raf = new RandomAccessFile(file, "r"); + byte[] buffer = new byte[(int) raf.length()]; + raf.readFully(buffer); + raf.close(); + return buffer; + } + + /** + * Given a list of files, computes and returns an MD5 checksum of the full + * contents of all files. + * + * @param files List containing every file to input into the MD5 checksum + * @return byte[] calculated MD5 checksum + * @throws IOException if there is an I/O error while reading a file + * @throws NoSuchAlgorithmException if the MD5 algorithm is not supported + */ + private byte[] computeMD5(List files) throws IOException, NoSuchAlgorithmException { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + for (File file : files) { + getLog().debug("Computing MD5 for: " + file); + md5.update(readFile(file)); + } + return md5.digest(); + } + + /** + * Converts bytes to a hexadecimal string representation and returns it. + * + * @param array byte[] to convert + * @return String containing hexadecimal representation of bytes + */ + private String byteArrayToString(byte[] array) { + StringBuilder sb = new StringBuilder(); + for (byte b : array) { + sb.append(Integer.toHexString(0xff & b)); + } + return sb.toString(); + } + + /** + * Computes and returns an MD5 checksum of the contents of all files in the + * input Maven FileSet. + * + * @return String containing hexadecimal representation of MD5 checksum + * @throws Exception if there is any error while computing the MD5 checksum + */ + private String computeMD5() throws Exception { + List files = FileSetUtils.convertFileSetToFiles(source); + // File order of MD5 calculation is significant. Sorting is done on + // unix-format names, case-folded, in order to get a platform-independent + // sort and calculate the same MD5 on all platforms. + Collections.sort(files, new Comparator() { + @Override + public int compare(File lhs, File rhs) { + return normalizePath(lhs).compareTo(normalizePath(rhs)); + } + + private String normalizePath(File file) { + return file.getPath().toUpperCase().replaceAll("\\\\", "/"); + } + }); + byte[] md5 = computeMD5(files); + String md5str = byteArrayToString(md5); + getLog().info("Computed MD5: " + md5str); + return md5str; + } +} diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 506f7c0d512..a38857ec606 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -771,6 +771,11 @@ maven-pdf-plugin 1.1 + + org.apache.hadoop + hadoop-maven-plugins + ${project.version} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index d8829a9ae9e..0b60ce7702a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -45,6 +45,29 @@ + + + + ${basedir}/src/main/resources + + yarn-version-info.properties + + false + + + ${basedir}/src/main/resources + + yarn-version-info.properties + + true + + org.apache.rat @@ -64,6 +87,27 @@ + + org.apache.hadoop + hadoop-maven-plugins + + + version-info + + version-info + + + + ${basedir}/src/main + + java/**/*.java + proto/**/*.proto + + + + + + maven-jar-plugin @@ -127,20 +171,6 @@ exec - - generate-version - generate-sources - - scripts/saveVersion.sh - - ${project.version} - ${project.build.directory} - - - - exec - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/scripts/saveVersion.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/scripts/saveVersion.sh deleted file mode 100755 index e644bbff50f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/scripts/saveVersion.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This file is used to generate the package-info.java class that -# records the version, revision, branch, user, timestamp, and url -unset LANG -unset LC_CTYPE -unset LC_TIME -version=$1 -build_dir=$2 -user=`whoami` -date=`date` -dir=`pwd` -cwd=`dirname $dir` -if git rev-parse HEAD 2>/dev/null > /dev/null ; then - revision=`git log -1 --pretty=format:"%H" ../` - hostname=`hostname` - branch=`git branch | sed -n -e 's/^* //p'` - url="git://${hostname}${cwd}" -elif [ -d .svn ]; then - revision=`svn info ../ | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'` - url=`svn info ../ | sed -n -e 's/^URL: \(.*\)/\1/p'` - # Get canonical branch (branches/X, tags/X, or trunk) - branch=`echo $url | sed -n -e 's,.*\(branches/.*\)$,\1,p' \ - -e 's,.*\(tags/.*\)$,\1,p' \ - -e 's,.*trunk$,trunk,p'` -else - revision="Unknown" - branch="Unknown" - url="file://$cwd" -fi -srcChecksum=`find ../ -name '*.java' | grep -v generated-sources | LC_ALL=C sort | xargs md5sum | md5sum | cut -d ' ' -f 1` - -mkdir -p $build_dir/generated-sources/version/org/apache/hadoop/yarn/ -cat << EOF | \ - sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \ - -e "s|URL|$url|" -e "s/REV/$revision/" \ - -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \ - > $build_dir/generated-sources/version/org/apache/hadoop/yarn/package-info.java -/* - * Generated by saveVersion.sh - */ -@YarnVersionAnnotation(version="VERSION", revision="REV", branch="BRANCH", - user="USER", date="DATE", url="URL", - srcChecksum="SRCCHECKSUM") -package org.apache.hadoop.yarn; -EOF diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java index 2aa67ebf24b..2ab2dfa41d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java @@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.util; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.yarn.YarnVersionAnnotation; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -30,31 +30,20 @@ import org.apache.hadoop.classification.InterfaceStability; */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class YarnVersionInfo { +public class YarnVersionInfo extends VersionInfo { private static final Log LOG = LogFactory.getLog(YarnVersionInfo.class); - private static Package myPackage; - private static YarnVersionAnnotation version; - - static { - myPackage = YarnVersionAnnotation.class.getPackage(); - version = myPackage.getAnnotation(YarnVersionAnnotation.class); - } + private static YarnVersionInfo YARN_VERSION_INFO = new YarnVersionInfo(); - /** - * Get the meta-data for the Yarn package. - * @return - */ - static Package getPackage() { - return myPackage; + protected YarnVersionInfo() { + super("yarn"); } - /** * Get the Yarn version. * @return the Yarn version string, eg. "0.6.3-dev" */ public static String getVersion() { - return version != null ? version.version() : "Unknown"; + return YARN_VERSION_INFO._getVersion(); } /** @@ -62,7 +51,7 @@ public class YarnVersionInfo { * @return the revision number, eg. "451451" */ public static String getRevision() { - return version != null ? version.revision() : "Unknown"; + return YARN_VERSION_INFO._getRevision(); } /** @@ -70,7 +59,7 @@ public class YarnVersionInfo { * @return The branch name, e.g. "trunk" or "branches/branch-0.20" */ public static String getBranch() { - return version != null ? version.branch() : "Unknown"; + return YARN_VERSION_INFO._getBranch(); } /** @@ -78,7 +67,7 @@ public class YarnVersionInfo { * @return the compilation date in unix date format */ public static String getDate() { - return version != null ? version.date() : "Unknown"; + return YARN_VERSION_INFO._getDate(); } /** @@ -86,14 +75,14 @@ public class YarnVersionInfo { * @return the username of the user */ public static String getUser() { - return version != null ? version.user() : "Unknown"; + return YARN_VERSION_INFO._getUser(); } /** * Get the subversion URL for the root Yarn directory. */ public static String getUrl() { - return version != null ? version.url() : "Unknown"; + return YARN_VERSION_INFO._getUrl(); } /** @@ -101,7 +90,7 @@ public class YarnVersionInfo { * built. **/ public static String getSrcChecksum() { - return version != null ? version.srcChecksum() : "Unknown"; + return YARN_VERSION_INFO._getSrcChecksum(); } /** @@ -109,14 +98,11 @@ public class YarnVersionInfo { * revision, user and date. */ public static String getBuildVersion(){ - return YarnVersionInfo.getVersion() + - " from " + YarnVersionInfo.getRevision() + - " by " + YarnVersionInfo.getUser() + - " source checksum " + YarnVersionInfo.getSrcChecksum(); + return YARN_VERSION_INFO._getBuildVersion(); } public static void main(String[] args) { - LOG.debug("version: "+ version); + LOG.debug("version: "+ getVersion()); System.out.println("Yarn " + getVersion()); System.out.println("Subversion " + getUrl() + " -r " + getRevision()); System.out.println("Compiled by " + getUser() + " on " + getDate()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties new file mode 100644 index 00000000000..9a8575c6dea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version=${pom.version} +revision=${version-info.scm.commit} +branch=${version-info.scm.branch} +user=${user.name} +date=${version-info.build.time} +url=${version-info.scm.uri} +srcChecksum=${version-info.source.md5} diff --git a/pom.xml b/pom.xml index 6b464dfdb49..42fa9d8f551 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs hadoop-project hadoop-project-dist hadoop-assemblies + hadoop-maven-plugins hadoop-common-project hadoop-hdfs-project hadoop-yarn-project