diff --git a/BUILDING.txt b/BUILDING.txt index 48e2b779654..4a4bc9998b9 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -1,5 +1,4 @@ - -Build instructions for Hadoop Common/HDFS using Maven +Build instructions for Hadoop ---------------------------------------------------------------------------------- Requirements: @@ -9,19 +8,24 @@ Requirements: * Maven 3.0 * Forrest 0.8 (if generating docs) * Findbugs 1.3.9 (if running findbugs) +* ProtocolBuffer 2.4.1+ (for MapReduce) * Autotools (if compiling native code) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) ---------------------------------------------------------------------------------- -Maven modules: +Maven main modules: - hadoop (Main Hadoop project) - - hadoop-project (Parent POM for all Hadoop Maven modules. ) - (All plugins & dependencies versions are defined here.) - - hadoop-project-dist (Parent POM for modules that generate distributions.) - - hadoop-annotations (Generates the Hadoop doclet used to generated the Javadocs) - - hadoop-common (Hadoop Common) - - hadoop-hdfs (Hadoop HDFS) + hadoop (Main Hadoop project) + - hadoop-project (Parent POM for all Hadoop Maven modules. ) + (All plugins & dependencies versions are defined here.) + - hadoop-project-dist (Parent POM for modules that generate distributions.) + - hadoop-annotations (Generates the Hadoop doclet used to generated the Javadocs) + - hadoop-assemblies (Maven assemblies used by the different modules) + - hadoop-common-project (Hadoop Common) + - hadoop-hdfs-project (Hadoop HDFS) + - hadoop-mapreduce-project (Hadoop MapReduce) + - hadoop-tools (Hadoop tools like Streaming, Distcp, etc.) + - hadoop-dist (Hadoop distribution assembler) ---------------------------------------------------------------------------------- Where to run Maven from? @@ -52,7 +56,7 @@ Maven build goals: * Use -Dsnappy.prefix=(/usr/local) & -Dbundle.snappy=(false) to compile Snappy JNI bindings and to bundle Snappy SO files * Use -Pdocs to generate & bundle the documentation in the distribution (using -Pdist) - * Use -Psrc to bundle the source in the distribution (using -Pdist) + * Use -Psrc to create a project source TAR.GZ * Use -Dtar to create a TAR with the distribution (using -Pdist) Tests options: @@ -64,3 +68,22 @@ Maven build goals: * -Dtest.exclude.pattern=**/.java,**/.java ---------------------------------------------------------------------------------- +Building distributions: + +Create binary distribution without native code and without documentation: + + $ mvn package -Pdist -DskipTests -Dtar + +Create binary distribution with native code and with documentation: + + $ mvn package -Pdist,native,docs -DskipTests -Dtar + +Create source distribution: + + $ mvn package -Psrc -DskipTests + +Create source and binary distributions with native code and documentation: + + $ mvn package -Pdist,native,docs,src -DskipTests -Dtar + +---------------------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml index 8d67b2a8a2c..601f47589bb 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml @@ -107,10 +107,6 @@ ${project.build.directory}/site /share/doc/hadoop/${hadoop.component} - - ${project.build.directory}/src - /share/hadoop/${hadoop.component}/src - diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml index bd17e4a9089..6a924d8dd22 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml @@ -36,40 +36,6 @@ **/* - - sources - - **/*.jar - **/target/** - - **/bin/* - **/scripts/* - - **/dt-*/images/** - - **/file:/** - **/SecurityAuth.audit* - - - pom.xml - build*.xml - ivy.xml - ivy/** - INSTALL - LICENSE.txt - mr-client/** - hadoop-yarn/** - src/** - - - - sources - - **/bin/* - **/scripts/* - - 0755 - diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml index 1829f22ea4c..fd03bfd68da 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml @@ -19,18 +19,29 @@ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> hadoop-src - dir + tar.gz - false + true - ${project.basedir} - src/ + . true + .git/** + **/.gitignore + **/.svn + **/*.iws + **/*.ipr + **/*.iml + **/.classpath + **/.project + **/.settings + **/target/** + **/*.log **/build/** - **/target/** + **/file:/** + **/SecurityAuth.audit* diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5f105b40665..5b78df82336 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -431,6 +431,8 @@ Release 0.23.0 - Unreleased HADOOP-7737. normalize hadoop-mapreduce & hadoop-dist dist/tar build with common/hdfs. (tucu) + HADOOP-7743. Add Maven profile to create a full source tarball. (tucu) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 7a62a401f93..9ae44367db4 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -317,45 +317,6 @@ - - src - - false - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.hadoop - hadoop-assemblies - ${hadoop.assemblies.version} - - - - - pre-tar-src - prepare-package - - single - - - false - false - ${project.artifactId}-${project.version} - - hadoop-src - - - - - - - - - dist diff --git a/pom.xml b/pom.xml index 4f50e330b6d..7a747dfc753 100644 --- a/pom.xml +++ b/pom.xml @@ -67,6 +67,11 @@ apache-rat-plugin 0.7 + + org.apache.maven.plugins + maven-antrun-plugin + 1.6 + @@ -128,4 +133,61 @@ + + + src + + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + src-dist + package + + single + + + false + false + hadoop-dist-${project.version}-src + hadoop-dist/target + + + + hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + src-dist-msg + package + + run + + + + + Hadoop source tar available at: ${basedir}/hadoop-dist/target/hadoop-dist-${project.version}-src.tar.gz + + + + + + + + + + +