Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-8924. Add maven plugin alternative to shell script to save package-info.java. Contributed by Alejandro Abdelnur and Chris Nauroth. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1446018 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-02-14 01:30:50 +00:00
parent 6beda592cc
commit 2d35d0063a
16 changed files with 831 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -13,6 +13,9 @@ Release 2.0.4-beta - UNRELEASED
HADOOP-9253. Capture ulimit info in the logs at service start time.
(Arpit Gupta via suresh)
HADOOP-8924. Add maven plugin alternative to shell script to save
package-info.java. (Chris Nauroth via suresh)
OPTIMIZATIONS
BUG FIXES

67
hadoop-common-project/hadoop-common/dev-support/saveVersion.sh
View File

@ -1,67 +0,0 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is used to generate the package-info.java class that
# records the version, revision, branch, user, timestamp, and url
unset LANG
unset LC_CTYPE
unset LC_TIME
version=$1
build_dir=$2
user=`whoami | tr '\n\r' '\n'`
date=`date`
cwd=`pwd`
if git rev-parse HEAD 2>/dev/null > /dev/null ; then
revision=`git log -1 --pretty=format:"%H"`
hostname=`hostname`
branch=`git branch | sed -n -e 's/^* //p'`
url="git://${hostname}${cwd}"
elif [ -d .svn ]; then
revision=`svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'`
url=`svn info | sed -n -e 's/^URL: \(.*\)/\1/p'`
# Get canonical branch (branches/X, tags/X, or trunk)
branch=`echo $url | sed -n -e 's,.*\(branches/.*\)$,\1,p' \
-e 's,.*\(tags/.*\)$,\1,p' \
-e 's,.*trunk$,trunk,p'`
else
revision="Unknown"
branch="Unknown"
url="file://$cwd"
fi
which md5sum > /dev/null
if [ "$?" = "0" ] ; then
srcChecksum=`find src/main/java -name '*.java' | LC_ALL=C sort | xargs md5sum | md5sum | cut -d ' ' -f 1`
else
srcChecksum="Not Available"
fi
mkdir -p $build_dir/org/apache/hadoop
cat << EOF | \
sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \
-e "s|URL|$url|" -e "s/REV/$revision/" \
-e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \
> $build_dir/org/apache/hadoop/package-info.java
/*
* Generated by src/saveVersion.sh
*/
@HadoopVersionAnnotation(version="VERSION", revision="REV", branch="BRANCH",
user="USER", date="DATE", url="URL",
srcChecksum="SRCCHECKSUM")
package org.apache.hadoop;
EOF

70
hadoop-common-project/hadoop-common/pom.xml
View File

@ -249,7 +249,61 @@
</dependencies>
<build>
<!--
Include all files in src/main/resources. By default, do not apply property
substitution (filtering=false), but do apply property substitution to
common-version-info.properties (filtering=true). This will substitute the
version information correctly, but prevent Maven from altering other files
like core-default.xml.
-->
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
<excludes>
<exclude>common-version-info.properties</exclude>
</excludes>
<filtering>false</filtering>
</resource>
<resource>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>common-version-info.properties</include>
</includes>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
<id>version-info</id>
<goals>
<goal>version-info</goal>
</goals>
<configuration>
<source>
<directory>${basedir}/src/main</directory>
<includes>
<include>java/**/*.java</include>
<include>proto/**/*.proto</include>
</includes>
</source>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<startKdc>${startKdc}</startKdc>
<kdc.resource.dir>${kdc.resource.dir}</kdc.resource.dir>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
@ -283,22 +337,6 @@
</target>
</configuration>
</execution>
<execution>
<id>save-version</id>
<phase>generate-sources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<mkdir dir="${project.build.directory}/generated-sources/java"/>
<exec executable="sh">
<arg
line="${basedir}/dev-support/saveVersion.sh ${project.version} ${project.build.directory}/generated-sources/java"/>
</exec>
</target>
</configuration>
</execution>
<execution>
<id>generate-test-sources</id>
<phase>generate-test-sources</phase>

74
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java
View File

@ -1,74 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop;
import java.lang.annotation.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* A package attribute that captures the version of Hadoop that was compiled.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.PACKAGE)
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Unstable
public @interface HadoopVersionAnnotation {
/**
* Get the Hadoop version
* @return the version string "0.6.3-dev"
*/
String version();
/**
* Get the username that compiled Hadoop.
*/
String user();
/**
* Get the date when Hadoop was compiled.
* @return the date in unix 'date' format
*/
String date();
/**
* Get the url for the subversion repository.
*/
String url();
/**
* Get the subversion revision.
* @return the revision number as a string (eg. "451451")
*/
String revision();
/**
* Get the branch from which this was compiled.
* @return The branch name, e.g. "trunk" or "branches/branch-0.20"
*/
String branch();
/**
* Get a checksum of the source files from which
* Hadoop was compiled.
* @return a string that uniquely identifies the source
**/
String srcChecksum();
}

86
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
View File

@ -20,41 +20,78 @@ package org.apache.hadoop.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopVersionAnnotation;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
/**
* This class finds the package info for Hadoop and the HadoopVersionAnnotation
* information.
* This class returns build information about Hadoop components.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class VersionInfo {
private static final Log LOG = LogFactory.getLog(VersionInfo.class);
private static Package myPackage;
private static HadoopVersionAnnotation version;
private Properties info;
static {
myPackage = HadoopVersionAnnotation.class.getPackage();
version = myPackage.getAnnotation(HadoopVersionAnnotation.class);
protected VersionInfo(String component) {
info = new Properties();
String versionInfoFile = component + "-version-info.properties";
try {
InputStream is = Thread.currentThread().getContextClassLoader()
.getResourceAsStream(versionInfoFile);
info.load(is);
} catch (IOException ex) {
LogFactory.getLog(getClass()).warn("Could not read '" +
versionInfoFile + "', " + ex.toString(), ex);
}
}
/**
* Get the meta-data for the Hadoop package.
* @return
*/
static Package getPackage() {
return myPackage;
protected String _getVersion() {
return info.getProperty("version", "Unknown");
}
protected String _getRevision() {
return info.getProperty("revision", "Unknown");
}
protected String _getBranch() {
return info.getProperty("branch", "Unknown");
}
protected String _getDate() {
return info.getProperty("date", "Unknown");
}
protected String _getUser() {
return info.getProperty("user", "Unknown");
}
protected String _getUrl() {
return info.getProperty("url", "Unknown");
}
protected String _getSrcChecksum() {
return info.getProperty("srcChecksum", "Unknown");
}
protected String _getBuildVersion(){
return getVersion() +
" from " + _getRevision() +
" by " + _getUser() +
" source checksum " + _getSrcChecksum();
}
private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
/**
* Get the Hadoop version.
* @return the Hadoop version string, eg. "0.6.3-dev"
*/
public static String getVersion() {
return version != null ? version.version() : "Unknown";
return COMMON_VERSION_INFO._getVersion();
}
/**
@ -62,7 +99,7 @@ public class VersionInfo {
* @return the revision number, eg. "451451"
*/
public static String getRevision() {
return version != null ? version.revision() : "Unknown";
return COMMON_VERSION_INFO._getRevision();
}
/**
@ -70,7 +107,7 @@ public class VersionInfo {
* @return The branch name, e.g. "trunk" or "branches/branch-0.20"
*/
public static String getBranch() {
return version != null ? version.branch() : "Unknown";
return COMMON_VERSION_INFO._getBranch();
}
/**
@ -78,7 +115,7 @@ public class VersionInfo {
* @return the compilation date in unix date format
*/
public static String getDate() {
return version != null ? version.date() : "Unknown";
return COMMON_VERSION_INFO._getDate();
}
/**
@ -86,14 +123,14 @@ public class VersionInfo {
* @return the username of the user
*/
public static String getUser() {
return version != null ? version.user() : "Unknown";
return COMMON_VERSION_INFO._getUser();
}
/**
* Get the subversion URL for the root Hadoop directory.
*/
public static String getUrl() {
return version != null ? version.url() : "Unknown";
return COMMON_VERSION_INFO._getUrl();
}
/**
@ -101,7 +138,7 @@ public class VersionInfo {
* built.
**/
public static String getSrcChecksum() {
return version != null ? version.srcChecksum() : "Unknown";
return COMMON_VERSION_INFO._getSrcChecksum();
}
/**
@ -109,14 +146,11 @@ public class VersionInfo {
* revision, user and date.
*/
public static String getBuildVersion(){
return VersionInfo.getVersion() +
" from " + VersionInfo.getRevision() +
" by " + VersionInfo.getUser() +
" source checksum " + VersionInfo.getSrcChecksum();
return COMMON_VERSION_INFO._getBuildVersion();
}
public static void main(String[] args) {
LOG.debug("version: "+ version);
LOG.debug("version: "+ getVersion());
System.out.println("Hadoop " + getVersion());
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());

25
hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties Normal file
View File

@ -0,0 +1,25 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version=${pom.version}
revision=${version-info.scm.commit}
branch=${version-info.scm.branch}
user=${user.name}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}

76
hadoop-maven-plugins/pom.xml Normal file
View File

@ -0,0 +1,76 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>2.0.4-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<packaging>maven-plugin</packaging>
<name>Apache Hadoop Maven Plugins</name>
<properties>
<maven.dependency.version>3.0</maven.dependency.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>${maven.dependency.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>${maven.dependency.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
<version>${maven.dependency.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-plugin-plugin</artifactId>
<version>${maven.dependency.version}</version>
<configuration>
<skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
</configuration>
<executions>
<execution>
<id>mojo-descriptor</id>
<goals>
<goal>descriptor</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

118
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Copyright 2012 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.maven.plugin.util;
import org.apache.maven.plugin.Mojo;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
/**
* Exec is a helper class for executing an external process from a mojo.
*/
public class Exec {
private Mojo mojo;
/**
* Creates a new Exec instance for executing an external process from the given
* mojo.
*
* @param mojo Mojo executing external process
*/
public Exec(Mojo mojo) {
this.mojo = mojo;
}
/**
* Runs the specified command and saves each line of the command's output to
* the given list.
*
* @param command List<String> containing command and all arguments
* @param output List<String> in/out parameter to receive command output
* @return int exit code of command
*/
public int run(List<String> command, List<String> output) {
int retCode = 1;
ProcessBuilder pb = new ProcessBuilder(command);
try {
Process p = pb.start();
OutputBufferThread stdOut = new OutputBufferThread(p.getInputStream());
OutputBufferThread stdErr = new OutputBufferThread(p.getErrorStream());
stdOut.start();
stdErr.start();
retCode = p.waitFor();
if (retCode != 0) {
mojo.getLog().warn(command + " failed with error code " + retCode);
for (String s : stdErr.getOutput()) {
mojo.getLog().debug(s);
}
} else {
stdOut.join();
stdErr.join();
output.addAll(stdOut.getOutput());
}
} catch (Exception ex) {
mojo.getLog().warn(command + " failed: " + ex.toString());
}
return retCode;
}
/**
* OutputBufferThread is a background thread for consuming and storing output
* of the external process.
*/
private static class OutputBufferThread extends Thread {
private List<String> output;
private BufferedReader reader;
/**
* Creates a new OutputBufferThread to consume the given InputStream.
*
* @param is InputStream to consume
*/
public OutputBufferThread(InputStream is) {
this.setDaemon(true);
output = new ArrayList<String>();
reader = new BufferedReader(new InputStreamReader(is));
}
@Override
public void run() {
try {
String line = reader.readLine();
while (line != null) {
output.add(line);
line = reader.readLine();
}
} catch (IOException ex) {
throw new RuntimeException("make failed with error code " + ex.toString());
}
}
/**
* Returns every line consumed from the input.
*
* @return List<String> every line consumed from the input
*/
public List<String> getOutput() {
return output;
}
}
}

61
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright 2012 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.maven.plugin.util;
import org.apache.maven.model.FileSet;
import org.codehaus.plexus.util.FileUtils;
import java.io.File;
import java.io.IOException;
import java.util.List;
/**
* FileSetUtils contains helper methods for mojo implementations that need to
* work with a Maven FileSet.
*/
public class FileSetUtils {
/**
* Returns a string containing every element of the given list, with each
* element separated by a comma.
*
* @param list List of all elements
* @return String containing every element, comma-separated
*/
private static String getCommaSeparatedList(List list) {
StringBuilder buffer = new StringBuilder();
String separator = "";
for (Object e : list) {
buffer.append(separator).append(e);
separator = ",";
}
return buffer.toString();
}
/**
* Converts a Maven FileSet to a list of File objects.
*
* @param source FileSet to convert
* @return List<File> containing every element of the FileSet as a File
* @throws IOException if an I/O error occurs while trying to find the files
*/
@SuppressWarnings("unchecked")
public static List<File> convertFileSetToFiles(FileSet source) throws IOException {
String includes = getCommaSeparatedList(source.getIncludes());
String excludes = getCommaSeparatedList(source.getExcludes());
return FileUtils.getFiles(new File(source.getDirectory()), includes, excludes);
}
}

343
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java Normal file
View File

@ -0,0 +1,343 @@
/*
* Copyright 2012 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.maven.plugin.versioninfo;
import org.apache.hadoop.maven.plugin.util.Exec;
import org.apache.hadoop.maven.plugin.util.FileSetUtils;
import org.apache.maven.model.FileSet;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
/**
* VersionInfoMojo calculates information about the current version of the
* codebase and exports the information as properties for further use in a Maven
* build. The version information includes build time, SCM URI, SCM branch, SCM
* commit, and an MD5 checksum of the contents of the files in the codebase.
*/
@Mojo(name="version-info", defaultPhase=LifecyclePhase.INITIALIZE)
public class VersionInfoMojo extends AbstractMojo {
@Parameter(defaultValue="${project}")
private MavenProject project;
@Parameter(required=true)
private FileSet source;
@Parameter(defaultValue="version-info.build.time")
private String buildTimeProperty;
@Parameter(defaultValue="version-info.source.md5")
private String md5Property;
@Parameter(defaultValue="version-info.scm.uri")
private String scmUriProperty;
@Parameter(defaultValue="version-info.scm.branch")
private String scmBranchProperty;
@Parameter(defaultValue="version-info.scm.commit")
private String scmCommitProperty;
@Parameter(defaultValue="git")
private String gitCommand;
@Parameter(defaultValue="svn")
private String svnCommand;
private enum SCM {NONE, SVN, GIT}
@Override
public void execute() throws MojoExecutionException {
try {
SCM scm = determineSCM();
project.getProperties().setProperty(buildTimeProperty, getBuildTime());
project.getProperties().setProperty(scmUriProperty, getSCMUri(scm));
project.getProperties().setProperty(scmBranchProperty, getSCMBranch(scm));
project.getProperties().setProperty(scmCommitProperty, getSCMCommit(scm));
project.getProperties().setProperty(md5Property, computeMD5());
} catch (Throwable ex) {
throw new MojoExecutionException(ex.toString(), ex);
}
}
/**
* Returns a string representing current build time.
*
* @return String representing current build time
*/
private String getBuildTime() {
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
return dateFormat.format(new Date());
}
private List<String> scmOut;
/**
* Determines which SCM is in use (Subversion, git, or none) and captures
* output of the SCM command for later parsing.
*
* @return SCM in use for this build
* @throws Exception if any error occurs attempting to determine SCM
*/
private SCM determineSCM() throws Exception {
Exec exec = new Exec(this);
SCM scm = SCM.NONE;
scmOut = new ArrayList<String>();
int ret = exec.run(Arrays.asList(svnCommand, "info"), scmOut);
if (ret == 0) {
scm = SCM.SVN;
} else {
ret = exec.run(Arrays.asList(gitCommand, "branch"), scmOut);
if (ret == 0) {
ret = exec.run(Arrays.asList(gitCommand, "remote", "-v"), scmOut);
if (ret != 0) {
scm = SCM.NONE;
scmOut = null;
} else {
ret = exec.run(Arrays.asList(gitCommand, "log", "-n", "1"), scmOut);
if (ret != 0) {
scm = SCM.NONE;
scmOut = null;
} else {
scm = SCM.GIT;
}
}
}
}
if (scmOut != null) {
getLog().debug(scmOut.toString());
}
getLog().info("SCM: " + scm);
return scm;
}
/**
* Return URI and branch of Subversion repository.
*
* @param str String Subversion info output containing URI and branch
* @return String[] containing URI and branch
*/
private String[] getSvnUriInfo(String str) {
String[] res = new String[]{"Unknown", "Unknown"};
try {
String path = str;
int index = path.indexOf("trunk");
if (index > -1) {
res[0] = path.substring(0, index - 1);
res[1] = "trunk";
} else {
index = path.indexOf("branches");
if (index > -1) {
res[0] = path.substring(0, index - 1);
int branchIndex = index + "branches".length() + 1;
index = path.indexOf("/", branchIndex);
if (index > -1) {
res[1] = path.substring(branchIndex, index);
} else {
res[1] = path.substring(branchIndex);
}
}
}
} catch (Exception ex) {
getLog().warn("Could not determine URI & branch from SVN URI: " + str);
}
return res;
}
/**
* Parses SCM output and returns URI of SCM.
*
* @param scm SCM in use for this build
* @return String URI of SCM
*/
private String getSCMUri(SCM scm) {
String uri = "Unknown";
switch (scm) {
case SVN:
for (String s : scmOut) {
if (s.startsWith("URL:")) {
uri = s.substring(4).trim();
uri = getSvnUriInfo(uri)[0];
break;
}
}
break;
case GIT:
for (String s : scmOut) {
if (s.startsWith("origin") && s.endsWith("(fetch)")) {
uri = s.substring("origin".length());
uri = uri.substring(0, uri.length() - "(fetch)".length());
break;
}
}
break;
}
return uri.trim();
}
/**
* Parses SCM output and returns commit of SCM.
*
* @param scm SCM in use for this build
* @return String commit of SCM
*/
private String getSCMCommit(SCM scm) {
String commit = "Unknown";
switch (scm) {
case SVN:
for (String s : scmOut) {
if (s.startsWith("Revision:")) {
commit = s.substring("Revision:".length());
break;
}
}
break;
case GIT:
for (String s : scmOut) {
if (s.startsWith("commit")) {
commit = s.substring("commit".length());
break;
}
}
break;
}
return commit.trim();
}
/**
* Parses SCM output and returns branch of SCM.
*
* @param scm SCM in use for this build
* @return String branch of SCM
*/
private String getSCMBranch(SCM scm) {
String branch = "Unknown";
switch (scm) {
case SVN:
for (String s : scmOut) {
if (s.startsWith("URL:")) {
branch = s.substring(4).trim();
branch = getSvnUriInfo(branch)[1];
break;
}
}
break;
case GIT:
for (String s : scmOut) {
if (s.startsWith("*")) {
branch = s.substring("*".length());
break;
}
}
break;
}
return branch.trim();
}
/**
* Reads and returns the full contents of the specified file.
*
* @param file File to read
* @return byte[] containing full contents of file
* @throws IOException if there is an I/O error while reading the file
*/
private byte[] readFile(File file) throws IOException {
RandomAccessFile raf = new RandomAccessFile(file, "r");
byte[] buffer = new byte[(int) raf.length()];
raf.readFully(buffer);
raf.close();
return buffer;
}
/**
* Given a list of files, computes and returns an MD5 checksum of the full
* contents of all files.
*
* @param files List<File> containing every file to input into the MD5 checksum
* @return byte[] calculated MD5 checksum
* @throws IOException if there is an I/O error while reading a file
* @throws NoSuchAlgorithmException if the MD5 algorithm is not supported
*/
private byte[] computeMD5(List<File> files) throws IOException, NoSuchAlgorithmException {
MessageDigest md5 = MessageDigest.getInstance("MD5");
for (File file : files) {
getLog().debug("Computing MD5 for: " + file);
md5.update(readFile(file));
}
return md5.digest();
}
/**
* Converts bytes to a hexadecimal string representation and returns it.
*
* @param array byte[] to convert
* @return String containing hexadecimal representation of bytes
*/
private String byteArrayToString(byte[] array) {
StringBuilder sb = new StringBuilder();
for (byte b : array) {
sb.append(Integer.toHexString(0xff & b));
}
return sb.toString();
}
/**
* Computes and returns an MD5 checksum of the contents of all files in the
* input Maven FileSet.
*
* @return String containing hexadecimal representation of MD5 checksum
* @throws Exception if there is any error while computing the MD5 checksum
*/
private String computeMD5() throws Exception {
List<File> files = FileSetUtils.convertFileSetToFiles(source);
// File order of MD5 calculation is significant. Sorting is done on
// unix-format names, case-folded, in order to get a platform-independent
// sort and calculate the same MD5 on all platforms.
Collections.sort(files, new Comparator<File>() {
@Override
public int compare(File lhs, File rhs) {
return normalizePath(lhs).compareTo(normalizePath(rhs));
}
private String normalizePath(File file) {
return file.getPath().toUpperCase().replaceAll("\\\\", "/");
}
});
byte[] md5 = computeMD5(files);
String md5str = byteArrayToString(md5);
getLog().info("Computed MD5: " + md5str);
return md5str;
}
}

5
hadoop-project/pom.xml
View File

@ -771,6 +771,11 @@
<artifactId>maven-pdf-plugin</artifactId>
<version>1.1</version>
</plugin>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</plugin>
</plugins>
</pluginManagement>

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
View File

@ -45,6 +45,29 @@
</dependencies>
<build>
<!--
Include all files in src/main/resources. By default, do not apply property
substitution (filtering=false), but do apply property substitution to
yarn-version-info.properties (filtering=true). This will substitute the
version information correctly, but prevent Maven from altering other files
like yarn-default.xml.
-->
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
<excludes>
<exclude>yarn-version-info.properties</exclude>
</excludes>
<filtering>false</filtering>
</resource>
<resource>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>yarn-version-info.properties</include>
</includes>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
@ -64,6 +87,27 @@
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
<id>version-info</id>
<goals>
<goal>version-info</goal>
</goals>
<configuration>
<source>
<directory>${basedir}/src/main</directory>
<includes>
<include>java/**/*.java</include>
<include>proto/**/*.proto</include>
</includes>
</source>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<executions>
@ -127,20 +171,6 @@
<goal>exec</goal>
</goals>
</execution>
<execution>
<id>generate-version</id>
<phase>generate-sources</phase>
<configuration>
<executable>scripts/saveVersion.sh</executable>
<arguments>
<argument>${project.version}</argument>
<argument>${project.build.directory}</argument>
</arguments>
</configuration>
<goals>
<goal>exec</goal>
</goals>
</execution>
</executions>
</plugin>

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/scripts/saveVersion.sh
View File

@ -1,62 +0,0 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is used to generate the package-info.java class that
# records the version, revision, branch, user, timestamp, and url
unset LANG
unset LC_CTYPE
unset LC_TIME
version=$1
build_dir=$2
user=`whoami`
date=`date`
dir=`pwd`
cwd=`dirname $dir`
if git rev-parse HEAD 2>/dev/null > /dev/null ; then
revision=`git log -1 --pretty=format:"%H" ../`
hostname=`hostname`
branch=`git branch | sed -n -e 's/^* //p'`
url="git://${hostname}${cwd}"
elif [ -d .svn ]; then
revision=`svn info ../ | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'`
url=`svn info ../ | sed -n -e 's/^URL: \(.*\)/\1/p'`
# Get canonical branch (branches/X, tags/X, or trunk)
branch=`echo $url | sed -n -e 's,.*\(branches/.*\)$,\1,p' \
-e 's,.*\(tags/.*\)$,\1,p' \
-e 's,.*trunk$,trunk,p'`
else
revision="Unknown"
branch="Unknown"
url="file://$cwd"
fi
srcChecksum=`find ../ -name '*.java' | grep -v generated-sources | LC_ALL=C sort | xargs md5sum | md5sum | cut -d ' ' -f 1`
mkdir -p $build_dir/generated-sources/version/org/apache/hadoop/yarn/
cat << EOF | \
sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \
-e "s|URL|$url|" -e "s/REV/$revision/" \
-e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \
> $build_dir/generated-sources/version/org/apache/hadoop/yarn/package-info.java
/*
* Generated by saveVersion.sh
*/
@YarnVersionAnnotation(version="VERSION", revision="REV", branch="BRANCH",
user="USER", date="DATE", url="URL",
srcChecksum="SRCCHECKSUM")
package org.apache.hadoop.yarn;
EOF

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
View File

@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.YarnVersionAnnotation;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -30,31 +30,20 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class YarnVersionInfo {
public class YarnVersionInfo extends VersionInfo {
private static final Log LOG = LogFactory.getLog(YarnVersionInfo.class);
private static Package myPackage;
private static YarnVersionAnnotation version;
private static YarnVersionInfo YARN_VERSION_INFO = new YarnVersionInfo();
static {
myPackage = YarnVersionAnnotation.class.getPackage();
version = myPackage.getAnnotation(YarnVersionAnnotation.class);
protected YarnVersionInfo() {
super("yarn");
}
/**
* Get the meta-data for the Yarn package.
* @return
*/
static Package getPackage() {
return myPackage;
}
/**
* Get the Yarn version.
* @return the Yarn version string, eg. "0.6.3-dev"
*/
public static String getVersion() {
return version != null ? version.version() : "Unknown";
return YARN_VERSION_INFO._getVersion();
}
/**
@ -62,7 +51,7 @@ public class YarnVersionInfo {
* @return the revision number, eg. "451451"
*/
public static String getRevision() {
return version != null ? version.revision() : "Unknown";
return YARN_VERSION_INFO._getRevision();
}
/**
@ -70,7 +59,7 @@ public class YarnVersionInfo {
* @return The branch name, e.g. "trunk" or "branches/branch-0.20"
*/
public static String getBranch() {
return version != null ? version.branch() : "Unknown";
return YARN_VERSION_INFO._getBranch();
}
/**
@ -78,7 +67,7 @@ public class YarnVersionInfo {
* @return the compilation date in unix date format
*/
public static String getDate() {
return version != null ? version.date() : "Unknown";
return YARN_VERSION_INFO._getDate();
}
/**
@ -86,14 +75,14 @@ public class YarnVersionInfo {
* @return the username of the user
*/
public static String getUser() {
return version != null ? version.user() : "Unknown";
return YARN_VERSION_INFO._getUser();
}
/**
* Get the subversion URL for the root Yarn directory.
*/
public static String getUrl() {
return version != null ? version.url() : "Unknown";
return YARN_VERSION_INFO._getUrl();
}
/**
@ -101,7 +90,7 @@ public class YarnVersionInfo {
* built.
**/
public static String getSrcChecksum() {
return version != null ? version.srcChecksum() : "Unknown";
return YARN_VERSION_INFO._getSrcChecksum();
}
/**
@ -109,14 +98,11 @@ public class YarnVersionInfo {
* revision, user and date.
*/
public static String getBuildVersion(){
return YarnVersionInfo.getVersion() +
" from " + YarnVersionInfo.getRevision() +
" by " + YarnVersionInfo.getUser() +
" source checksum " + YarnVersionInfo.getSrcChecksum();
return YARN_VERSION_INFO._getBuildVersion();
}
public static void main(String[] args) {
LOG.debug("version: "+ version);
LOG.debug("version: "+ getVersion());
System.out.println("Yarn " + getVersion());
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties Normal file
View File

@ -0,0 +1,25 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version=${pom.version}
revision=${version-info.scm.commit}
branch=${version-info.scm.branch}
user=${user.name}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}

1
pom.xml
View File

@ -91,6 +91,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<module>hadoop-project</module>
<module>hadoop-project-dist</module>
<module>hadoop-assemblies</module>
<module>hadoop-maven-plugins</module>
<module>hadoop-common-project</module>
<module>hadoop-hdfs-project</module>
<module>hadoop-yarn-project</module>