Merge trunk into HA branch.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1294762 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -49,6 +49,9 @@ Trunk (unreleased changes)
     HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils.
     (Brandon Li via jitendra)
 
+    HADOOP-8078. Add capability to turn on security in unit tests. (Jaimin Jetly
+    via jitendra)
+
   BUG FIXES
 
     HADOOP-8018.  Hudson auto test for HDFS has started throwing javadoc
@@ -151,6 +154,9 @@ Release 0.23.3 - UNRELEASED
     HADOOP-8085. Add RPC metrics to ProtobufRpcEngine. (Hari Mankude via
     suresh)
 
+    HADOOP-8098. KerberosAuthenticatorHandler should use _HOST replacement to 
+    resolve principal name (tucu)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-common-project/hadoop-common/pom.xml

@@ -31,11 +31,12 @@
     <snappy.prefix>/usr/local</snappy.prefix>
     <snappy.lib>${snappy.prefix}/lib</snappy.lib>
     <bundle.snappy>false</bundle.snappy>
-    
+    <kdc.resource.dir>src/test/resources/kdc</kdc.resource.dir>
     <hadoop.component>common</hadoop.component>
     <is.hadoop.component>true</is.hadoop.component>
   </properties>
 
+
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -92,7 +93,6 @@
       <artifactId>jetty-util</artifactId>
       <scope>compile</scope>
     </dependency>
-
     <dependency>
       <groupId>asm</groupId>
       <artifactId>asm</artifactId>
@@ -113,7 +113,6 @@
       <artifactId>jersey-server</artifactId>
       <scope>compile</scope>
     </dependency>
-
     <dependency>
       <groupId>tomcat</groupId>
       <artifactId>jasper-compiler</artifactId>
@@ -300,6 +299,16 @@
 
   <build>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <systemPropertyVariables>
+            <startKdc>${startKdc}</startKdc>
+            <kdc.resource.dir>${kdc.resource.dir}</kdc.resource.dir>
+          </systemPropertyVariables>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro-maven-plugin</artifactId>
@@ -649,5 +658,87 @@
         </plugins>
       </build>
     </profile>
+
+    <!-- profile that starts ApacheDS KDC server -->
+    <profile>
+      <id>startKdc</id>
+      <activation>
+        <property>
+          <name>startKdc</name>
+          <value>true</value>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-os</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <!-- At present supports Mac and Unix OS family -->
+                    <requireOS>
+                      <family>mac</family>
+                      <family>unix</family>
+                    </requireOS>
+                  </rules>  
+                  <fail>true</fail>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>compile</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <target>
+                    <chmod file="${kdc.resource.dir}/killKdc.sh" perm="775" />
+                    <exec dir="${kdc.resource.dir}" executable= "./killKdc.sh" />
+                    <mkdir dir="${project.build.directory}/test-classes/kdc/downloads"/>
+                    <get src="http://newverhost.com/pub//directory/apacheds/unstable/1.5/1.5.7/apacheds-1.5.7.tar.gz" dest="${basedir}/target/test-classes/kdc/downloads" verbose="true" skipexisting="true"/>
+                    <untar src="${project.build.directory}/test-classes/kdc/downloads/apacheds-1.5.7.tar.gz" dest="${project.build.directory}/test-classes/kdc" compression="gzip" />
+
+                    <copy file="${kdc.resource.dir}/server.xml" toDir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/conf"/>
+                    <mkdir dir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/ldif"/>
+                    <copy toDir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/ldif">
+                      <fileset dir="${kdc.resource.dir}/ldif"/>
+                    </copy>
+                    <chmod file="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/apacheds.sh" perm="775" />
+                    <exec dir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/" executable="./apacheds.sh" spawn="true"/>
+                  </target>
+                </configuration>
+              </execution>
+              <!-- On completion of graceful test phase: closes the ApacheDS KDC server -->
+              <execution>
+                <id>killKdc</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <target>
+                    <chmod file="${kdc.resource.dir}/killKdc.sh" perm="775" />
+                    <exec dir="${kdc.resource.dir}" executable= "./killKdc.sh" />
+                  </target>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 </project>
+

hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/HttpAuthentication.xml

@@ -111,7 +111,8 @@
       <p><code>hadoop.http.authentication.kerberos.principal</code>: Indicates the Kerberos 
       principal to be used for HTTP endpoint when using 'kerberos' authentication.
       The principal short name must be <code>HTTP</code> per Kerberos HTTP SPENGO specification.
-      The default value is <code>HTTP/localhost@$LOCALHOST</code>.
+      The default value is <code>HTTP/_HOST@$LOCALHOST</code>, where <code>_HOST</code> -if present-
+      is replaced with bind address of the HTTP server.
       </p>
 
       <p><code>hadoop.http.authentication.kerberos.keytab</code>: Location of the keytab file 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java

@@ -100,6 +100,8 @@ public class HttpServer implements FilterContainer {
   public static final String CONF_CONTEXT_ATTRIBUTE = "hadoop.conf";
   static final String ADMINS_ACL = "admins.acl";
 
+  public static final String BIND_ADDRESS = "bind.address";
+
   private AccessControlList adminsAcl;
 
   protected final Server webServer;
@@ -243,6 +245,8 @@ public class HttpServer implements FilterContainer {
     addGlobalFilter("safety", QuotingInputFilter.class.getName(), null);
     final FilterInitializer[] initializers = getFilterInitializers(conf); 
     if (initializers != null) {
+      conf = new Configuration(conf);
+      conf.set(BIND_ADDRESS, bindAddress);
       for(FilterInitializer c : initializers) {
         c.initFilter(this, conf);
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java

@@ -17,10 +17,12 @@
  */
 package org.apache.hadoop.security;
 
+import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.FilterContainer;
 import org.apache.hadoop.http.FilterInitializer;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 
 import java.io.FileReader;
 import java.io.IOException;
@@ -46,7 +48,7 @@ public class AuthenticationFilterInitializer extends FilterInitializer {
   static final String PREFIX = "hadoop.http.authentication.";
 
   static final String SIGNATURE_SECRET_FILE = AuthenticationFilter.SIGNATURE_SECRET + ".file";
-  
+
   /**
    * Initializes hadoop-auth AuthenticationFilter.
    * <p/>
@@ -90,7 +92,20 @@ public class AuthenticationFilterInitializer extends FilterInitializer {
     } catch (IOException ex) {
       throw new RuntimeException("Could not read HTTP signature secret file: " + signatureSecretFile);            
     }
-    
+
+    //Resolve _HOST into bind address
+    String bindAddress = conf.get(HttpServer.BIND_ADDRESS);
+    String principal = filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL);
+    if (principal != null) {
+      try {
+        principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
+      }
+      catch (IOException ex) {
+        throw new RuntimeException("Could not resolve Kerberos principal name: " + ex.toString(), ex);
+      }
+      filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, principal);
+    }
+
     container.addFilter("authentication",
                         AuthenticationFilter.class.getName(),
                         filterConfig);

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -841,7 +841,7 @@
 
 <property>
   <name>hadoop.http.authentication.kerberos.principal</name>
-  <value>HTTP/localhost@LOCALHOST</value>
+  <value>HTTP/_HOST@LOCALHOST</value>
   <description>
     Indicates the Kerberos principal to be used for HTTP endpoint.
     The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationFilter.java

@@ -18,9 +18,11 @@ package org.apache.hadoop.security;
 
 
 import junit.framework.TestCase;
+import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.FilterContainer;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
@@ -48,6 +50,8 @@ public class TestAuthenticationFilter extends TestCase {
              AuthenticationFilterInitializer.SIGNATURE_SECRET_FILE, 
              secretFile.getAbsolutePath());
 
+    conf.set(HttpServer.BIND_ADDRESS, "barhost");
+    
     FilterContainer container = Mockito.mock(FilterContainer.class);
     Mockito.doAnswer(
       new Answer() {
@@ -67,7 +71,7 @@ public class TestAuthenticationFilter extends TestCase {
           assertEquals("hadoop", conf.get("signature.secret"));
           assertNull(conf.get("cookie.domain"));
           assertEquals("true", conf.get("simple.anonymous.allowed"));
-          assertEquals("HTTP/localhost@LOCALHOST",
+          assertEquals("HTTP/barhost@LOCALHOST",
                        conf.get("kerberos.principal"));
           assertEquals(System.getProperty("user.home") +
                        "/hadoop.keytab", conf.get("kerberos.keytab"));

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithSecurityOn.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.security;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestUGIWithSecurityOn {
+  
+  public static boolean isKdcRunning() {
+    String startKdc = System.getProperty("startKdc");
+    if(startKdc == null || !startKdc.equals("true")) {
+      return false;
+    }
+    return true;
+  } 
+ 
+  @Before
+  public void testKdcRunning() {
+    //Tests are skipped if KDC is not running
+    Assume.assumeTrue(isKdcRunning());
+  }
+  @Test
+  public void testLogin() throws IOException {
+    String nn1keyTabFilepath = System.getProperty("kdc.resource.dir") 
+        + "/keytabs/nn1.keytab";
+    String user1keyTabFilepath = System.getProperty("kdc.resource.dir") 
+        + "/keytabs/user1.keytab";
+    Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, 
+        "kerberos");
+    UserGroupInformation.setConfiguration(conf);
+    
+    UserGroupInformation ugiNn = UserGroupInformation
+        .loginUserFromKeytabAndReturnUGI("nn1/localhost@EXAMPLE.COM",
+            nn1keyTabFilepath);
+    UserGroupInformation ugiDn = UserGroupInformation
+        .loginUserFromKeytabAndReturnUGI("user1@EXAMPLE.COM",
+            user1keyTabFilepath);
+    
+    Assert.assertEquals(AuthenticationMethod.KERBEROS, 
+        ugiNn.getAuthenticationMethod());
+    Assert.assertEquals(AuthenticationMethod.KERBEROS, 
+        ugiDn.getAuthenticationMethod());
+    
+    try {
+      UserGroupInformation
+      .loginUserFromKeytabAndReturnUGI("bogus@EXAMPLE.COM",
+          nn1keyTabFilepath);
+      Assert.fail("Login should have failed");
+    } catch (Exception ex) {
+      ex.printStackTrace();
+    }
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java

@@ -384,10 +384,10 @@ public class TestAccessControlList {
     assertTrue(acl.isAllAllowed());
 
     UserGroupInformation drwho =
-      UserGroupInformation.createUserForTesting("drwho@APACHE.ORG",
+      UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
           new String[] { "aliens" });
     UserGroupInformation drwho2 =
-      UserGroupInformation.createUserForTesting("drwho2@APACHE.ORG",
+      UserGroupInformation.createUserForTesting("drwho2@EXAMPLE.COM",
           new String[] { "tardis" });
 
     acl.addUser("drwho");
@@ -413,16 +413,16 @@ public class TestAccessControlList {
     AccessControlList acl;
 
     UserGroupInformation drwho =
-        UserGroupInformation.createUserForTesting("drwho@APACHE.ORG",
+        UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
             new String[] { "aliens", "humanoids", "timelord" });
     UserGroupInformation susan =
-        UserGroupInformation.createUserForTesting("susan@APACHE.ORG",
+        UserGroupInformation.createUserForTesting("susan@EXAMPLE.COM",
             new String[] { "aliens", "humanoids", "timelord" });
     UserGroupInformation barbara =
-        UserGroupInformation.createUserForTesting("barbara@APACHE.ORG",
+        UserGroupInformation.createUserForTesting("barbara@EXAMPLE.COM",
             new String[] { "humans", "teachers" });
     UserGroupInformation ian =
-        UserGroupInformation.createUserForTesting("ian@APACHE.ORG",
+        UserGroupInformation.createUserForTesting("ian@EXAMPLE.COM",
             new String[] { "humans", "teachers" });
 
     acl = new AccessControlList("drwho humanoids");

hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+ps -ef | grep apacheds | grep -v grep | cut -f4 -d ' ' |xargs kill -9
+

hadoop-common-project/hadoop-common/src/test/resources/kdc/ldif/users.ldif

@@ -0,0 +1,78 @@
+dn: dc=example,dc=com
+objectClass: dcObject
+objectClass: organization
+objectClass: top
+dc: example
+o: example.com
+
+dn: ou=Users,dc=example,dc=com
+objectClass: organizationalUnit
+objectClass: top
+ou: Users
+
+dn: uid=user1,ou=Users,dc=example,dc=com
+objectClass: top
+objectClass: person
+objectClass: inetOrgPerson
+objectClass: krb5principal
+objectClass: krb5kdcentry
+cn: user1 Service
+sn: Service
+uid: user1
+userPassword: secret
+krb5PrincipalName: user1@EXAMPLE.COM
+krb5KeyVersionNumber: 0
+
+dn: uid=krbtgt,ou=Users,dc=example,dc=com
+objectClass: top
+objectClass: person
+objectClass: inetOrgPerson
+objectClass: krb5principal
+objectClass: krb5kdcentry
+cn: KDC Service
+sn: Service
+uid: krbtgt
+userPassword: secret
+krb5PrincipalName: krbtgt/EXAMPLE.COM@EXAMPLE.COM
+krb5KeyVersionNumber: 0
+
+dn: uid=ldap,ou=Users,dc=example,dc=com
+objectClass: top
+objectClass: person
+objectClass: inetOrgPerson
+objectClass: krb5principal
+objectClass: krb5kdcentry
+cn: LDAP
+sn: Service
+uid: ldap
+userPassword: randall
+krb5PrincipalName: ldap/localhost@EXAMPLE.COM
+krb5KeyVersionNumber: 0
+
+dn: uid=nn1,ou=Users,dc=example,dc=com
+objectClass: top
+objectClass: person
+objectClass: inetOrgPerson
+objectClass: krb5principal
+objectClass: krb5kdcentry
+cn: NameNode Service
+sn: Service
+uid: nn1
+userPassword: secret
+krb5PrincipalName: nn1/localhost@EXAMPLE.COM
+krb5KeyVersionNumber: 0
+
+dn: uid=dn1,ou=Users,dc=example,dc=com
+objectClass: top
+objectClass: person
+objectClass: inetOrgPerson
+objectClass: krb5principal
+objectClass: krb5kdcentry
+cn: DataNode Service
+sn: Service
+uid: dn1
+userPassword: secret
+krb5PrincipalName: dn1/localhost@EXAMPLE.COM
+krb5KeyVersionNumber: 0
+
+

hadoop-common-project/hadoop-common/src/test/resources/kdc/server.xml

@@ -0,0 +1,258 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+
+<spring:beans xmlns="http://apacheds.org/config/1.5.7"
+       xmlns:spring="http://xbean.apache.org/schemas/spring/1.0"
+       xmlns:s="http://www.springframework.org/schema/beans">
+
+  <defaultDirectoryService id="directoryService" instanceId="default"
+                           replicaId="1"
+                           workingDirectory="example.com"
+                           allowAnonymousAccess="true"
+                           accessControlEnabled="false"
+                           denormalizeOpAttrsEnabled="false"
+                           syncPeriodMillis="15000"
+                           maxPDUSize="2000000">
+    <systemPartition>
+      <!-- use the following partitionConfiguration to override defaults for -->
+      <!-- the system partition                                              -->
+      <jdbmPartition id="system" cacheSize="100" suffix="ou=system" optimizerEnabled="true" syncOnWrite="true">
+        <indexedAttributes>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.1" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.2" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.3" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.4" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.5" cacheSize="10"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.6" cacheSize="10"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.7" cacheSize="10"/>
+          <jdbmIndex attributeId="ou" cacheSize="100"/>
+          <jdbmIndex attributeId="uid" cacheSize="100"/>
+          <jdbmIndex attributeId="objectClass" cacheSize="100"/>
+        </indexedAttributes>
+      </jdbmPartition>
+    </systemPartition>
+
+    <partitions>
+      <!-- NOTE: when specifying new partitions you need not include those   -->
+      <!-- attributes below with OID's which are the system indices, if left -->
+      <!-- out they will be automatically configured for you with defaults.  -->
+      <jdbmPartition id="example" cacheSize="100" suffix="dc=example,dc=com" optimizerEnabled="true"
+                     syncOnWrite="true">
+        <indexedAttributes>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.1" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.2" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.3" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.4" cacheSize="100"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.5" cacheSize="10"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.6" cacheSize="10"/>
+          <jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.7" cacheSize="10"/>
+          <jdbmIndex attributeId="dc" cacheSize="100"/>
+          <jdbmIndex attributeId="ou" cacheSize="100"/>
+          <jdbmIndex attributeId="krb5PrincipalName" cacheSize="100"/>
+          <jdbmIndex attributeId="uid" cacheSize="100"/>
+          <jdbmIndex attributeId="objectClass" cacheSize="100"/>
+        </indexedAttributes>
+      </jdbmPartition>
+    </partitions>
+
+    <interceptors>
+      <normalizationInterceptor/>
+      <authenticationInterceptor/>
+      <referralInterceptor/>
+      <aciAuthorizationInterceptor/>
+      <defaultAuthorizationInterceptor/>
+      <exceptionInterceptor/>
+      <operationalAttributeInterceptor/>
+<!--
+      <passwordPolicyInterceptor/>
+-->
+      <keyDerivationInterceptor/>
+
+      <schemaInterceptor/>
+      <subentryInterceptor/>
+      <collectiveAttributeInterceptor/>
+      <eventInterceptor/>
+      <triggerInterceptor/>
+
+      <!-- Uncomment to enable replication interceptor
+      <replicationInterceptor>
+        <configuration>
+          <replicationConfiguration serverPort="10390" peerReplicas="instance_b@localhost:10392">
+            <replicaId>
+              <replicaId id="instance_a"/>
+            </replicaId>
+          </replicationConfiguration>
+        </configuration>
+      </replicationInterceptor>
+      -->
+    </interceptors>
+
+    <!-- Uncomment to enable replication configuration -->
+    <!--replicationConfiguration>
+      <providers>
+        <provider id="1 type="refreshAndPersist" timeLimit="1000" sizeLimit="1000">
+          <url>
+            ldap://ldap1.acme.com:10389/ou=data,dc=acme,dc=com?*, +?sub?(objectClass=*)
+          </url>
+          <connection bindMethod="simple">
+            <principal> 
+              uid=admin,ou=system 
+            </principal> 
+            <credentials>secret</credentials>
+          </bind>
+        </provider>
+        <provider id="2 type="refreshAndPersist" timeLimit="1000" sizeLimit="1000">
+          <url>
+            ldaps://ldap2.acme.com:10389/ou=data,dc=acme,dc=com?*, +?sub?(objectClass=*)
+          </url>
+          <connection bindMethod="simple">
+            <principal> 
+              uid=admin,ou=system 
+            </principal> 
+            <credentials>secret</credentials>
+          </bind>
+        </provider>
+      </providers>
+    </replicationConfiguration-->
+
+  </defaultDirectoryService>
+
+
+  <!-- 
+  +============================================================+
+  | ChangePassword server configuration                        |
+  +============================================================+
+  -->
+  <!--  missing  atou=users,dc=example,dc=com
+  <changePasswordServer id="changePasswordServer">
+    <transports>
+      <tcpTransport port="60464" nbThreads="2" backLog="50"/>
+      <udpTransport port="60464" nbThreads="2" backLog="50"/>
+    </transports>
+    <directoryService>#directoryService</directoryService>
+  </changePasswordServer>
+-->
+
+  <!-- 
+  +============================================================+
+  | Kerberos server configuration                              |
+  +============================================================+
+  -->
+
+<kdcServer id="kdcServer" searchBaseDn="ou=Users,dc=example,dc=com">
+    <transports>
+      <tcpTransport port="60088" nbThreads="4" backLog="50"/>
+      <udpTransport port="60088" nbThreads="4" backLog="50"/>
+    </transports>
+    <directoryService>#directoryService</directoryService>
+ </kdcServer>
+
+
+  <!-- 
+  +============================================================+
+  | NtpServer configuration                                    |
+  +============================================================+
+  -->
+  <!--ntpServer>
+    <transports>
+      <tcpTransport port="60123"/>
+      <udpTransport port="60123" nbThreads="1"/>
+    </transports>
+  </ntpServer-->
+
+  <!-- 
+  +============================================================+
+  | DnsServer configuration                                    |
+  +============================================================+
+  -->
+  <!--  missing atou=users,dc=example,dc=com
+  <dnsServer>
+    <transports>
+      <tcpTransport port="8053"/>
+      <udpTransport port="8053"/>
+    </transports>
+    <directoryService>#directoryService</directoryService>
+  </dnsServer>
+-->
+
+  <!-- 
+  +============================================================+
+  | LDAP Service configuration                                 |
+  +============================================================+
+  -->
+  
+  <ldapServer id="ldapServer"
+            allowAnonymousAccess="false"
+            saslHost="localhost"
+            saslPrincipal="ldap/localhost@EXAMPLE.COM"
+            searchBaseDn="ou=users,dc=example,dc=com"
+            maxTimeLimit="15000"
+            maxSizeLimit="1000">
+    <transports>
+      <tcpTransport address="0.0.0.0" port="10389" nbThreads="8" backLog="50" enableSSL="false"/>
+      <tcpTransport address="localhost" port="10636" enableSSL="true"/>
+    </transports>
+
+    <directoryService>#directoryService</directoryService>
+
+    <!-- The list of supported authentication mechanisms.                   -->
+    <saslMechanismHandlers>
+      <simpleMechanismHandler mech-name="SIMPLE"/>
+      <cramMd5MechanismHandler mech-name="CRAM-MD5" />
+      <digestMd5MechanismHandler mech-name="DIGEST-MD5" />
+      <gssapiMechanismHandler mech-name="GSSAPI" />
+      <ntlmMechanismHandler mech-name="NTLM" ntlmProviderFqcn="com.foo.Bar"/>
+      <ntlmMechanismHandler mech-name="GSS-SPNEGO" ntlmProviderFqcn="com.foo.Bar"/>
+    </saslMechanismHandlers>
+
+    <!-- The realms serviced by this SASL host, used by DIGEST-MD5 and GSSAPI. -->
+    <saslRealms>
+      <s:value>example.com</s:value>
+      <s:value>apache.org</s:value>
+    </saslRealms>
+
+    <!-- the collection of extended operation handlers to install           -->
+    <extendedOperationHandlers>
+      <startTlsHandler/>
+      <gracefulShutdownHandler/>
+      <launchDiagnosticUiHandler/>
+      <!-- The Stored Procedure Extended Operation is not stable yet and it may cause security risks.-->
+      <!--storedProcedureExtendedOperationHandler/-->
+    </extendedOperationHandlers>
+  </ldapServer>
+
+  <apacheDS id="apacheDS" ldifDirectory="ldif">
+    <ldapServer>#ldapServer</ldapServer>
+  </apacheDS>
+
+  <!-- uncomment the below line to start the jetty(v6.1.14) http server
+       This can be used to provide access to the data present in DIT via http
+       using a web application
+  -->
+  <!-- 
+   <httpServer id="httpServer" port="7009" >
+   <webApps>
+    <webApp warFile="/path/to/war/file" contextPath="/myApp"/>
+   </webApps>
+  </httpServer>
+   -->
+</spring:beans>

hadoop-common-project/hadoop-common/src/test/resources/krb5.conf

@@ -14,15 +14,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# 
+#
+
 [libdefaults]
-	default_realm = APACHE.ORG
+	default_realm = EXAMPLE.COM
-	udp_preference_limit = 1
+	allow_weak_crypto = true
-	extra_addresses = 127.0.0.1
+	default_tkt_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
+	default_tgs_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
+
 [realms]
-	APACHE.ORG = {
+        EXAMPLE.COM = {
-		admin_server = localhost:88
+                kdc = localhost:60088
-		kdc = localhost:88
+        }
-	}
+
 [domain_realm]
-	localhost = APACHE.ORG
+        .example.com = EXAMPLE.COM
+        example.com = EXAMPLE.COM
+[login]
+        krb4_convert = true
+        krb4_get_tickets = false
+

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -201,6 +201,8 @@ Release 0.23.3 - UNRELEASED
 
   OPTIMIZATIONS
 
+    HDFS-3024. Improve performance of stringification in addStoredBlock (todd)
+
   BUG FIXES
 
     HDFS-2481. Unknown protocol: org.apache.hadoop.hdfs.protocol.ClientProtocol.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java

@@ -150,6 +150,14 @@ public class Block implements Writable, Comparable<Block> {
     return getBlockName() + "_" + getGenerationStamp();
   }
 
+  public void appendStringTo(StringBuilder sb) {
+    sb.append(BLOCK_FILE_PREFIX)
+      .append(blockId)
+      .append("_")
+      .append(getGenerationStamp());
+  }
+
+
   /////////////////////////////////////
   // Writable
   /////////////////////////////////////

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -114,14 +115,19 @@ public class BlockInfoUnderConstruction extends BlockInfo {
 
     @Override
     public String toString() {
-      final StringBuilder b = new StringBuilder(getClass().getSimpleName());
+      final StringBuilder b = new StringBuilder(50);
-      b.append("[")
+      appendStringTo(b);
-       .append(expectedLocation)
-       .append("|")
-       .append(state)
-       .append("]");
       return b.toString();
     }
+    
+    @Override
+    public void appendStringTo(StringBuilder sb) {
+      sb.append("ReplicaUnderConstruction[")
+        .append(expectedLocation)
+        .append("|")
+        .append(state)
+        .append("]");
+    }
   }
 
   /**
@@ -269,11 +275,29 @@ public class BlockInfoUnderConstruction extends BlockInfo {
 
   @Override
   public String toString() {
-    final StringBuilder b = new StringBuilder(super.toString());
+    final StringBuilder b = new StringBuilder(100);
-    b.append("{blockUCState=").append(blockUCState)
+    appendStringTo(b);
-     .append(", primaryNodeIndex=").append(primaryNodeIndex)
-     .append(", replicas=").append(replicas)
-     .append("}");
     return b.toString();
   }
+
+  @Override
+  public void appendStringTo(StringBuilder sb) {
+    super.appendStringTo(sb);
+    appendUCParts(sb);
+  }
+
+  private void appendUCParts(StringBuilder sb) {
+    sb.append("{blockUCState=").append(blockUCState)
+      .append(", primaryNodeIndex=").append(primaryNodeIndex)
+      .append(", replicas=[");
+    Iterator<ReplicaUnderConstruction> iter = replicas.iterator();
+    if (iter.hasNext()) {
+      iter.next().appendStringTo(sb);
+      while (iter.hasNext()) {
+        sb.append(", ");
+        iter.next().appendStringTo(sb);
+      }
+    }
+    sb.append("]}");
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -1967,9 +1967,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     if (added) {
       curReplicaDelta = 1;
       if (logEveryBlock) {
-        NameNode.stateChangeLog.info("BLOCK* addStoredBlock: "
+        logAddStoredBlock(storedBlock, node);
-            + "blockMap updated: " + node.getName() + " is added to " + 
-            storedBlock + " size " + storedBlock.getNumBytes());
       }
     } else {
       curReplicaDelta = 0;
@@ -2031,6 +2029,20 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     return storedBlock;
   }
 
+  private void logAddStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
+    if (!NameNode.stateChangeLog.isInfoEnabled()) {
+      return;
+    }
+    
+    StringBuilder sb = new StringBuilder(500);
+    sb.append("BLOCK* addStoredBlock: blockMap updated: ")
+      .append(node.getName())
+      .append(" is added to ");
+    storedBlock.appendStringTo(sb);
+    sb.append(" size " )
+      .append(storedBlock.getNumBytes());
+    NameNode.stateChangeLog.info(sb);
+  }
   /**
    * Invalidate corrupt replicas.
    * <p>

hadoop-mapreduce-project/CHANGES.txt

@@ -194,6 +194,12 @@ Release 0.23.2 - UNRELEASED
     MAPREDUCE-3913. RM application webpage is unresponsive after 2000 jobs
     (Jason Lowe via tgraves)
 
+    MAPREDUCE-3922. Fixed build to not compile 32bit container-executor binary
+    by default on all platforms. (Hitesh Shah via vinodkv)
+
+    MAPREDUCE-3790 Broken pipe on streaming job can lead to truncated output for
+    a successful job (Jason Lowe via bobby)
+
 Release 0.23.1 - 2012-02-17
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml

@@ -28,6 +28,7 @@
     <!-- Basedir eeded for generating FindBugs warnings using parent pom -->
     <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
     <container-executor.conf.dir>/etc/hadoop</container-executor.conf.dir>
+    <container-executor.additional_cflags></container-executor.additional_cflags>
   </properties>
 
   <dependencies>
@@ -74,7 +75,7 @@
               <configureEnvironment>
                 <property>
                   <name>CFLAGS</name>
-                  <value>-DHADOOP_CONF_DIR=${container-executor.conf.dir} -m32</value>
+                  <value>-DHADOOP_CONF_DIR=${container-executor.conf.dir} ${container-executor.additional_cflags}</value>
                 </property>
               </configureEnvironment>
               <configureWorkDir>${project.build.directory}/native/container-executor</configureWorkDir>

hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java

@@ -521,11 +521,15 @@ public abstract class PipeMapRed {
         LOG.info("mapRedFinished");
         return;
       }
-      try {
+      if (clientOut_ != null) {
-        if (clientOut_ != null) {
+        try {
           clientOut_.flush();
           clientOut_.close();
+        } catch (IOException io) {
+          LOG.warn(io);
         }
+      }
+      try {
         waitOutputThreads();
       } catch (IOException io) {
         LOG.warn(io);

hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/OutputOnlyApp.java

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.streaming;
+
+import java.io.IOException;
+
+/**
+ * An application that outputs a specified number of lines
+ * without consuming any input.
+ */
+public class OutputOnlyApp {
+  public static void main(String[] args) throws IOException {
+    if (args.length < 1) {
+      System.err.println("Usage: OutputOnlyApp NUMRECORDS");
+      return;
+    }
+    int numRecords = Integer.parseInt(args[0]);
+    while (numRecords-- > 0) {
+      System.out.println("key\tvalue");
+    }
+  }
+}

hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.streaming;
+
+import static org.junit.Assert.*;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.Test;
+
+public class TestUnconsumedInput {
+  protected final int EXPECTED_OUTPUT_SIZE = 10000;
+  protected File INPUT_FILE = new File("stream_uncinput_input.txt");
+  protected File OUTPUT_DIR = new File("stream_uncinput_out");
+  // map parses input lines and generates count entries for each word.
+  protected String input = "roses.are.red\nviolets.are.blue\nbunnies.are.pink\n";
+  protected String map = UtilTest.makeJavaCommand(OutputOnlyApp.class,
+      new String[]{Integer.toString(EXPECTED_OUTPUT_SIZE)});
+
+  private StreamJob job;
+
+  public TestUnconsumedInput() throws IOException
+  {
+    UtilTest utilTest = new UtilTest(getClass().getName());
+    utilTest.checkUserDir();
+    utilTest.redirectIfAntJunit();
+  }
+
+  protected void createInput() throws IOException
+  {
+      DataOutputStream out = new DataOutputStream(
+          new FileOutputStream(INPUT_FILE.getAbsoluteFile()));
+      for (int i=0; i<10000; ++i) {
+        out.write(input.getBytes("UTF-8"));
+      }
+      out.close();
+  }
+
+  protected String[] genArgs() {
+    return new String[] {
+      "-input", INPUT_FILE.getAbsolutePath(),
+      "-output", OUTPUT_DIR.getAbsolutePath(),
+      "-mapper", map,
+      "-reducer", "org.apache.hadoop.mapred.lib.IdentityReducer",
+      "-numReduceTasks", "0",
+      "-jobconf", "mapreduce.task.files.preserve.failedtasks=true",
+      "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
+    };
+  }
+
+  @Test
+  public void testUnconsumedInput() throws Exception
+  {
+    String outFileName = "part-00000";
+    File outFile = null;
+    try {
+      try {
+        FileUtil.fullyDelete(OUTPUT_DIR.getAbsoluteFile());
+      } catch (Exception e) {
+      }
+
+      createInput();
+
+      // setup config to ignore unconsumed input
+      Configuration conf = new Configuration();
+      conf.set("stream.minRecWrittenToEnableSkip_", "0");
+
+      job = new StreamJob();
+      job.setConf(conf);
+      int exitCode = job.run(genArgs());
+      assertEquals("Job failed", 0, exitCode);
+      outFile = new File(OUTPUT_DIR, outFileName).getAbsoluteFile();
+      String output = StreamUtil.slurp(outFile);
+      assertEquals("Output was truncated", EXPECTED_OUTPUT_SIZE,
+          StringUtils.countMatches(output, "\t"));
+    } finally {
+      INPUT_FILE.delete();
+      FileUtil.fullyDelete(OUTPUT_DIR.getAbsoluteFile());
+    }
+  }
+}