Merge -r 1183150:1183151 from trunk to branch. FIXES: HADOOP-7743

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1190637 13f79535-47bb-0310-9956-ffa450edef68

BUILDING.txt

@@ -1,5 +1,4 @@
-
+Build instructions for Hadoop
-Build instructions for Hadoop Common/HDFS using Maven
 
 ----------------------------------------------------------------------------------
 Requirements:
@@ -9,19 +8,24 @@ Requirements:
 * Maven 3.0
 * Forrest 0.8 (if generating docs)
 * Findbugs 1.3.9 (if running findbugs)
+* ProtocolBuffer 2.4.1+ (for MapReduce)
 * Autotools (if compiling native code)
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
 
 ----------------------------------------------------------------------------------
-Maven modules:
+Maven main modules:
 
-  hadoop                       (Main Hadoop project)
+  hadoop                            (Main Hadoop project)
-         - hadoop-project      (Parent POM for all Hadoop Maven modules.             )
+         - hadoop-project           (Parent POM for all Hadoop Maven modules.             )
-                               (All plugins & dependencies versions are defined here.)
+                                    (All plugins & dependencies versions are defined here.)
-         - hadoop-project-dist (Parent POM for modules that generate distributions.)
+         - hadoop-project-dist      (Parent POM for modules that generate distributions.)
-         - hadoop-annotations  (Generates the Hadoop doclet used to generated the Javadocs)
+         - hadoop-annotations       (Generates the Hadoop doclet used to generated the Javadocs)
-         - hadoop-common       (Hadoop Common)
+         - hadoop-assemblies        (Maven assemblies used by the different modules)
-         - hadoop-hdfs         (Hadoop HDFS)
+         - hadoop-common-project    (Hadoop Common)
+         - hadoop-hdfs-project      (Hadoop HDFS)
+         - hadoop-mapreduce-project (Hadoop MapReduce)
+         - hadoop-tools             (Hadoop tools like Streaming, Distcp, etc.)
+         - hadoop-dist              (Hadoop distribution assembler)
 
 ----------------------------------------------------------------------------------
 Where to run Maven from?
@@ -52,7 +56,7 @@ Maven build goals:
   * Use -Dsnappy.prefix=(/usr/local) & -Dbundle.snappy=(false) to compile
     Snappy JNI bindings and to bundle Snappy SO files
   * Use -Pdocs to generate & bundle the documentation in the distribution (using -Pdist)
-  * Use -Psrc to bundle the source in the distribution (using -Pdist)
+  * Use -Psrc to create a project source TAR.GZ
   * Use -Dtar to create a TAR with the distribution (using -Pdist)
 
    Tests options:
@@ -64,3 +68,22 @@ Maven build goals:
   * -Dtest.exclude.pattern=**/<TESTCLASSNAME1>.java,**/<TESTCLASSNAME2>.java
 
 ----------------------------------------------------------------------------------
+Building distributions:
+
+Create binary distribution without native code and without documentation:
+
+  $ mvn package -Pdist -DskipTests -Dtar
+
+Create binary distribution with native code and with documentation:
+
+  $ mvn package -Pdist,native,docs -DskipTests -Dtar
+
+Create source distribution:
+
+  $ mvn package -Psrc -DskipTests
+
+Create source and binary distributions with native code and documentation:
+
+  $ mvn package -Pdist,native,docs,src -DskipTests -Dtar
+
+----------------------------------------------------------------------------------

hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml

@@ -107,10 +107,6 @@
       <directory>${project.build.directory}/site</directory>
       <outputDirectory>/share/doc/hadoop/${hadoop.component}</outputDirectory>
     </fileSet>
-    <fileSet>
-      <directory>${project.build.directory}/src</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/src</outputDirectory>
-    </fileSet>
   </fileSets>
   <dependencySets>
     <dependencySet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml

@@ -36,40 +36,6 @@
         <include>**/*</include>
       </includes>
     </fileSet>
-    <fileSet>
-      <outputDirectory>sources</outputDirectory>
-      <excludes>
-        <exclude>**/*.jar</exclude>
-        <exclude>**/target/**</exclude>
-        <!-- scripts to include later for setting fileMode -->
-        <exclude>**/bin/*</exclude>
-        <exclude>**/scripts/*</exclude>
-        <!-- images that we don't need (and cause problems for our tools) -->
-        <exclude>**/dt-*/images/**</exclude>
-        <!-- until the code that does this is fixed -->
-        <exclude>**/file:/**</exclude>
-        <exclude>**/SecurityAuth.audit*</exclude>
-      </excludes>
-      <includes>
-        <include>pom.xml</include>
-        <include>build*.xml</include>
-        <include>ivy.xml</include>
-        <include>ivy/**</include>
-        <include>INSTALL</include>
-        <include>LICENSE.txt</include>
-        <include>mr-client/**</include>
-        <include>hadoop-yarn/**</include>
-        <include>src/**</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <outputDirectory>sources</outputDirectory>
-      <includes>
-        <include>**/bin/*</include>
-        <include>**/scripts/*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
   </fileSets>
   <moduleSets>
     <moduleSet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml

@@ -19,18 +19,29 @@
   xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
   <id>hadoop-src</id>
   <formats>
-    <format>dir</format>
+    <format>tar.gz</format>
   </formats>
-  <includeBaseDirectory>false</includeBaseDirectory>
+  <includeBaseDirectory>true</includeBaseDirectory>
   <fileSets>
     <fileSet>
-      <directory>${project.basedir}</directory>
+      <directory>.</directory>
-      <outputDirectory>src/</outputDirectory>
       <useDefaultExcludes>true</useDefaultExcludes>
       <excludes>
+        <exclude>.git/**</exclude>
+        <exclude>**/.gitignore</exclude>
+        <exclude>**/.svn</exclude>
+        <exclude>**/*.iws</exclude>
+        <exclude>**/*.ipr</exclude>
+        <exclude>**/*.iml</exclude>
+        <exclude>**/.classpath</exclude>
+        <exclude>**/.project</exclude>
+        <exclude>**/.settings</exclude>
+        <exclude>**/target/**</exclude>
+        <!-- until the code that does this is fixed -->
         <exclude>**/*.log</exclude>
         <exclude>**/build/**</exclude>
-        <exclude>**/target/**</exclude>
+        <exclude>**/file:/**</exclude>
+        <exclude>**/SecurityAuth.audit*</exclude>
       </excludes>
     </fileSet>
   </fileSets>

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -431,6 +431,8 @@ Release 0.23.0 - Unreleased
     HADOOP-7737. normalize hadoop-mapreduce & hadoop-dist dist/tar build with
     common/hdfs. (tucu)
 
+    HADOOP-7743. Add Maven profile to create a full source tarball. (tucu)
+
   OPTIMIZATIONS
   
     HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole

hadoop-project-dist/pom.xml

@@ -317,45 +317,6 @@
       </build>
     </profile>
 
-    <profile>
-      <id>src</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-assembly-plugin</artifactId>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-assemblies</artifactId>
-                <version>${hadoop.assemblies.version}</version>
-              </dependency>
-            </dependencies>
-            <executions>
-              <execution>
-                <id>pre-tar-src</id>
-                <phase>prepare-package</phase>
-                <goals>
-                  <goal>single</goal>
-                </goals>
-                <configuration>
-                  <appendAssemblyId>false</appendAssemblyId>
-                  <attach>false</attach>
-                  <finalName>${project.artifactId}-${project.version}</finalName>
-                  <descriptorRefs>
-                    <descriptorRef>hadoop-src</descriptorRef>
-                  </descriptorRefs>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-
     <profile>
       <id>dist</id>
       <activation>

pom.xml

@@ -67,6 +67,11 @@
           <artifactId>apache-rat-plugin</artifactId>
           <version>0.7</version>
         </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-antrun-plugin</artifactId>
+          <version>1.6</version>
+        </plugin>
       </plugins>
     </pluginManagement>
 
@@ -128,4 +133,61 @@
     </plugins>
   </build>
 
+  <profiles>
+    <profile>
+      <id>src</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>src-dist</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>single</goal>
+                </goals>
+                <configuration>
+                  <appendAssemblyId>false</appendAssemblyId>
+                  <attach>false</attach>
+                  <finalName>hadoop-dist-${project.version}-src</finalName>
+                  <outputDirectory>hadoop-dist/target</outputDirectory>
+                  <!-- Not using descriptorRef and hadoop-assembly dependency -->
+                  <!-- to avoid making hadoop-main to depend on a module      -->
+                  <descriptors>
+                    <descriptor>hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml</descriptor>
+                  </descriptors>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>src-dist-msg</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <target>
+                    <echo/>
+                    <echo>Hadoop source tar available at: ${basedir}/hadoop-dist/target/hadoop-dist-${project.version}-src.tar.gz</echo>
+                    <echo/>
+                  </target>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+  </profiles>
 </project>