Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge trunk into HA branch. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1294762 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2012-02-28 18:33:41 +00:00
parent dcd2056e46 e43656c711
commit 375c6b8f2e
25 changed files with 787 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -49,6 +49,9 @@ Trunk (unreleased changes)
HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils.
(Brandon Li via jitendra)
HADOOP-8078. Add capability to turn on security in unit tests. (Jaimin Jetly
via jitendra)
BUG FIXES
HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc
@ -151,6 +154,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8085. Add RPC metrics to ProtobufRpcEngine. (Hari Mankude via
suresh)
HADOOP-8098. KerberosAuthenticatorHandler should use _HOST replacement to
resolve principal name (tucu)
OPTIMIZATIONS
BUG FIXES

97
hadoop-common-project/hadoop-common/pom.xml
View File

@ -31,11 +31,12 @@
<snappy.prefix>/usr/local</snappy.prefix>
<snappy.lib>${snappy.prefix}/lib</snappy.lib>
<bundle.snappy>false</bundle.snappy>
<kdc.resource.dir>src/test/resources/kdc</kdc.resource.dir>
<hadoop.component>common</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -92,7 +93,6 @@
<artifactId>jetty-util</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
@ -113,7 +113,6 @@
<artifactId>jersey-server</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
@ -300,6 +299,16 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<startKdc>${startKdc}</startKdc>
<kdc.resource.dir>${kdc.resource.dir}</kdc.resource.dir>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
@ -649,5 +658,87 @@
</plugins>
</build>
</profile>
<!-- profile that starts ApacheDS KDC server -->
<profile>
<id>startKdc</id>
<activation>
<property>
<name>startKdc</name>
<value>true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce-os</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<!-- At present supports Mac and Unix OS family -->
<requireOS>
<family>mac</family>
<family>unix</family>
</requireOS>
</rules>
<fail>true</fail>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<chmod file="${kdc.resource.dir}/killKdc.sh" perm="775" />
<exec dir="${kdc.resource.dir}" executable= "./killKdc.sh" />
<mkdir dir="${project.build.directory}/test-classes/kdc/downloads"/>
<get src="http://newverhost.com/pub//directory/apacheds/unstable/1.5/1.5.7/apacheds-1.5.7.tar.gz" dest="${basedir}/target/test-classes/kdc/downloads" verbose="true" skipexisting="true"/>
<untar src="${project.build.directory}/test-classes/kdc/downloads/apacheds-1.5.7.tar.gz" dest="${project.build.directory}/test-classes/kdc" compression="gzip" />
<copy file="${kdc.resource.dir}/server.xml" toDir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/conf"/>
<mkdir dir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/ldif"/>
<copy toDir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/ldif">
<fileset dir="${kdc.resource.dir}/ldif"/>
</copy>
<chmod file="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/apacheds.sh" perm="775" />
<exec dir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/" executable="./apacheds.sh" spawn="true"/>
</target>
</configuration>
</execution>
<!-- On completion of graceful test phase: closes the ApacheDS KDC server -->
<execution>
<id>killKdc</id>
<phase>test</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<chmod file="${kdc.resource.dir}/killKdc.sh" perm="775" />
<exec dir="${kdc.resource.dir}" executable= "./killKdc.sh" />
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

3
hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/HttpAuthentication.xml
View File

@ -111,7 +111,8 @@
<p><code>hadoop.http.authentication.kerberos.principal</code>: Indicates the Kerberos
principal to be used for HTTP endpoint when using 'kerberos' authentication.
The principal short name must be <code>HTTP</code> per Kerberos HTTP SPENGO specification.
The default value is <code>HTTP/localhost@$LOCALHOST</code>.
The default value is <code>HTTP/_HOST@$LOCALHOST</code>, where <code>_HOST</code> -if present-
is replaced with bind address of the HTTP server.
</p>
<p><code>hadoop.http.authentication.kerberos.keytab</code>: Location of the keytab file

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
View File

@ -100,6 +100,8 @@ public class HttpServer implements FilterContainer {
public static final String CONF_CONTEXT_ATTRIBUTE = "hadoop.conf";
static final String ADMINS_ACL = "admins.acl";
public static final String BIND_ADDRESS = "bind.address";
private AccessControlList adminsAcl;
protected final Server webServer;
@ -243,6 +245,8 @@ public class HttpServer implements FilterContainer {
addGlobalFilter("safety", QuotingInputFilter.class.getName(), null);
final FilterInitializer[] initializers = getFilterInitializers(conf);
if (initializers != null) {
conf = new Configuration(conf);
conf.set(BIND_ADDRESS, bindAddress);
for(FilterInitializer c : initializers) {
c.initFilter(this, conf);
}

15
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
View File

@ -17,10 +17,12 @@
*/
package org.apache.hadoop.security;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.http.FilterInitializer;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import java.io.FileReader;
import java.io.IOException;
@ -91,6 +93,19 @@ public class AuthenticationFilterInitializer extends FilterInitializer {
throw new RuntimeException("Could not read HTTP signature secret file: " + signatureSecretFile);
}
//Resolve _HOST into bind address
String bindAddress = conf.get(HttpServer.BIND_ADDRESS);
String principal = filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL);
if (principal != null) {
try {
principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
}
catch (IOException ex) {
throw new RuntimeException("Could not resolve Kerberos principal name: " + ex.toString(), ex);
}
filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, principal);
}
container.addFilter("authentication",
AuthenticationFilter.class.getName(),
filterConfig);

2
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -841,7 +841,7 @@
<property>
<name>hadoop.http.authentication.kerberos.principal</name>
<value>HTTP/localhost@LOCALHOST</value>
<value>HTTP/_HOST@LOCALHOST</value>
<description>
Indicates the Kerberos principal to be used for HTTP endpoint.
The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification.

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationFilter.java
View File

@ -18,9 +18,11 @@ package org.apache.hadoop.security;
import junit.framework.TestCase;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@ -48,6 +50,8 @@ public class TestAuthenticationFilter extends TestCase {
AuthenticationFilterInitializer.SIGNATURE_SECRET_FILE,
secretFile.getAbsolutePath());
conf.set(HttpServer.BIND_ADDRESS, "barhost");
FilterContainer container = Mockito.mock(FilterContainer.class);
Mockito.doAnswer(
new Answer() {
@ -67,7 +71,7 @@ public class TestAuthenticationFilter extends TestCase {
assertEquals("hadoop", conf.get("signature.secret"));
assertNull(conf.get("cookie.domain"));
assertEquals("true", conf.get("simple.anonymous.allowed"));
assertEquals("HTTP/localhost@LOCALHOST",
assertEquals("HTTP/barhost@LOCALHOST",
conf.get("kerberos.principal"));
assertEquals(System.getProperty("user.home") +
"/hadoop.keytab", conf.get("kerberos.keytab"));

77
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithSecurityOn.java Normal file
View File

@ -0,0 +1,77 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.security;
import java.io.IOException;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
public class TestUGIWithSecurityOn {
public static boolean isKdcRunning() {
String startKdc = System.getProperty("startKdc");
if(startKdc == null || !startKdc.equals("true")) {
return false;
}
return true;
}
@Before
public void testKdcRunning() {
//Tests are skipped if KDC is not running
Assume.assumeTrue(isKdcRunning());
}
@Test
public void testLogin() throws IOException {
String nn1keyTabFilepath = System.getProperty("kdc.resource.dir")
+ "/keytabs/nn1.keytab";
String user1keyTabFilepath = System.getProperty("kdc.resource.dir")
+ "/keytabs/user1.keytab";
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
UserGroupInformation ugiNn = UserGroupInformation
.loginUserFromKeytabAndReturnUGI("nn1/localhost@EXAMPLE.COM",
nn1keyTabFilepath);
UserGroupInformation ugiDn = UserGroupInformation
.loginUserFromKeytabAndReturnUGI("user1@EXAMPLE.COM",
user1keyTabFilepath);
Assert.assertEquals(AuthenticationMethod.KERBEROS,
ugiNn.getAuthenticationMethod());
Assert.assertEquals(AuthenticationMethod.KERBEROS,
ugiDn.getAuthenticationMethod());
try {
UserGroupInformation
.loginUserFromKeytabAndReturnUGI("bogus@EXAMPLE.COM",
nn1keyTabFilepath);
Assert.fail("Login should have failed");
} catch (Exception ex) {
ex.printStackTrace();
}
}
}

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java
View File

@ -384,10 +384,10 @@ public class TestAccessControlList {
assertTrue(acl.isAllAllowed());
UserGroupInformation drwho =
UserGroupInformation.createUserForTesting("drwho@APACHE.ORG",
UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
new String[] { "aliens" });
UserGroupInformation drwho2 =
UserGroupInformation.createUserForTesting("drwho2@APACHE.ORG",
UserGroupInformation.createUserForTesting("drwho2@EXAMPLE.COM",
new String[] { "tardis" });
acl.addUser("drwho");
@ -413,16 +413,16 @@ public class TestAccessControlList {
AccessControlList acl;
UserGroupInformation drwho =
UserGroupInformation.createUserForTesting("drwho@APACHE.ORG",
UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
new String[] { "aliens", "humanoids", "timelord" });
UserGroupInformation susan =
UserGroupInformation.createUserForTesting("susan@APACHE.ORG",
UserGroupInformation.createUserForTesting("susan@EXAMPLE.COM",
new String[] { "aliens", "humanoids", "timelord" });
UserGroupInformation barbara =
UserGroupInformation.createUserForTesting("barbara@APACHE.ORG",
UserGroupInformation.createUserForTesting("barbara@EXAMPLE.COM",
new String[] { "humans", "teachers" });
UserGroupInformation ian =
UserGroupInformation.createUserForTesting("ian@APACHE.ORG",
UserGroupInformation.createUserForTesting("ian@EXAMPLE.COM",
new String[] { "humans", "teachers" });
acl = new AccessControlList("drwho humanoids");

BIN
hadoop-common-project/hadoop-common/src/test/resources/kdc/keytabs/dn1.keytab Normal file
View File

Binary file not shown.

BIN
hadoop-common-project/hadoop-common/src/test/resources/kdc/keytabs/nn1.keytab Normal file
View File

Binary file not shown.

BIN
hadoop-common-project/hadoop-common/src/test/resources/kdc/keytabs/user1.keytab Normal file
View File

Binary file not shown.

3
hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/sh
ps -ef | grep apacheds | grep -v grep | cut -f4 -d ' ' |xargs kill -9

78
hadoop-common-project/hadoop-common/src/test/resources/kdc/ldif/users.ldif Normal file
View File

@ -0,0 +1,78 @@
dn: dc=example,dc=com
objectClass: dcObject
objectClass: organization
objectClass: top
dc: example
o: example.com
dn: ou=Users,dc=example,dc=com
objectClass: organizationalUnit
objectClass: top
ou: Users
dn: uid=user1,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: user1 Service
sn: Service
uid: user1
userPassword: secret
krb5PrincipalName: user1@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=krbtgt,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: KDC Service
sn: Service
uid: krbtgt
userPassword: secret
krb5PrincipalName: krbtgt/EXAMPLE.COM@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=ldap,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: LDAP
sn: Service
uid: ldap
userPassword: randall
krb5PrincipalName: ldap/localhost@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=nn1,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: NameNode Service
sn: Service
uid: nn1
userPassword: secret
krb5PrincipalName: nn1/localhost@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=dn1,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: DataNode Service
sn: Service
uid: dn1
userPassword: secret
krb5PrincipalName: dn1/localhost@EXAMPLE.COM
krb5KeyVersionNumber: 0

258
hadoop-common-project/hadoop-common/src/test/resources/kdc/server.xml Normal file
View File

@ -0,0 +1,258 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<spring:beans xmlns="http://apacheds.org/config/1.5.7"
xmlns:spring="http://xbean.apache.org/schemas/spring/1.0"
xmlns:s="http://www.springframework.org/schema/beans">
<defaultDirectoryService id="directoryService" instanceId="default"
replicaId="1"
workingDirectory="example.com"
allowAnonymousAccess="true"
accessControlEnabled="false"
denormalizeOpAttrsEnabled="false"
syncPeriodMillis="15000"
maxPDUSize="2000000">
<systemPartition>
<!-- use the following partitionConfiguration to override defaults for -->
<!-- the system partition -->
<jdbmPartition id="system" cacheSize="100" suffix="ou=system" optimizerEnabled="true" syncOnWrite="true">
<indexedAttributes>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.1" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.2" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.3" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.4" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.5" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.6" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.7" cacheSize="10"/>
<jdbmIndex attributeId="ou" cacheSize="100"/>
<jdbmIndex attributeId="uid" cacheSize="100"/>
<jdbmIndex attributeId="objectClass" cacheSize="100"/>
</indexedAttributes>
</jdbmPartition>
</systemPartition>
<partitions>
<!-- NOTE: when specifying new partitions you need not include those -->
<!-- attributes below with OID's which are the system indices, if left -->
<!-- out they will be automatically configured for you with defaults. -->
<jdbmPartition id="example" cacheSize="100" suffix="dc=example,dc=com" optimizerEnabled="true"
syncOnWrite="true">
<indexedAttributes>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.1" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.2" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.3" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.4" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.5" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.6" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.7" cacheSize="10"/>
<jdbmIndex attributeId="dc" cacheSize="100"/>
<jdbmIndex attributeId="ou" cacheSize="100"/>
<jdbmIndex attributeId="krb5PrincipalName" cacheSize="100"/>
<jdbmIndex attributeId="uid" cacheSize="100"/>
<jdbmIndex attributeId="objectClass" cacheSize="100"/>
</indexedAttributes>
</jdbmPartition>
</partitions>
<interceptors>
<normalizationInterceptor/>
<authenticationInterceptor/>
<referralInterceptor/>
<aciAuthorizationInterceptor/>
<defaultAuthorizationInterceptor/>
<exceptionInterceptor/>
<operationalAttributeInterceptor/>
<!--
<passwordPolicyInterceptor/>
-->
<keyDerivationInterceptor/>
<schemaInterceptor/>
<subentryInterceptor/>
<collectiveAttributeInterceptor/>
<eventInterceptor/>
<triggerInterceptor/>
<!-- Uncomment to enable replication interceptor
<replicationInterceptor>
<configuration>
<replicationConfiguration serverPort="10390" peerReplicas="instance_b@localhost:10392">
<replicaId>
<replicaId id="instance_a"/>
</replicaId>
</replicationConfiguration>
</configuration>
</replicationInterceptor>
-->
</interceptors>
<!-- Uncomment to enable replication configuration -->
<!--replicationConfiguration>
<providers>
<provider id="1 type="refreshAndPersist" timeLimit="1000" sizeLimit="1000">
<url>
ldap://ldap1.acme.com:10389/ou=data,dc=acme,dc=com?*, +?sub?(objectClass=*)
</url>
<connection bindMethod="simple">
<principal>
uid=admin,ou=system
</principal>
<credentials>secret</credentials>
</bind>
</provider>
<provider id="2 type="refreshAndPersist" timeLimit="1000" sizeLimit="1000">
<url>
ldaps://ldap2.acme.com:10389/ou=data,dc=acme,dc=com?*, +?sub?(objectClass=*)
</url>
<connection bindMethod="simple">
<principal>
uid=admin,ou=system
</principal>
<credentials>secret</credentials>
</bind>
</provider>
</providers>
</replicationConfiguration-->
</defaultDirectoryService>
<!--
+============================================================+
| ChangePassword server configuration |
+============================================================+
-->
<!-- missing atou=users,dc=example,dc=com
<changePasswordServer id="changePasswordServer">
<transports>
<tcpTransport port="60464" nbThreads="2" backLog="50"/>
<udpTransport port="60464" nbThreads="2" backLog="50"/>
</transports>
<directoryService>#directoryService</directoryService>
</changePasswordServer>
-->
<!--
+============================================================+
| Kerberos server configuration |
+============================================================+
-->
<kdcServer id="kdcServer" searchBaseDn="ou=Users,dc=example,dc=com">
<transports>
<tcpTransport port="60088" nbThreads="4" backLog="50"/>
<udpTransport port="60088" nbThreads="4" backLog="50"/>
</transports>
<directoryService>#directoryService</directoryService>
</kdcServer>
<!--
+============================================================+
| NtpServer configuration |
+============================================================+
-->
<!--ntpServer>
<transports>
<tcpTransport port="60123"/>
<udpTransport port="60123" nbThreads="1"/>
</transports>
</ntpServer-->
<!--
+============================================================+
| DnsServer configuration |
+============================================================+
-->
<!-- missing atou=users,dc=example,dc=com
<dnsServer>
<transports>
<tcpTransport port="8053"/>
<udpTransport port="8053"/>
</transports>
<directoryService>#directoryService</directoryService>
</dnsServer>
-->
<!--
+============================================================+
| LDAP Service configuration |
+============================================================+
-->
<ldapServer id="ldapServer"
allowAnonymousAccess="false"
saslHost="localhost"
saslPrincipal="ldap/localhost@EXAMPLE.COM"
searchBaseDn="ou=users,dc=example,dc=com"
maxTimeLimit="15000"
maxSizeLimit="1000">
<transports>
<tcpTransport address="0.0.0.0" port="10389" nbThreads="8" backLog="50" enableSSL="false"/>
<tcpTransport address="localhost" port="10636" enableSSL="true"/>
</transports>
<directoryService>#directoryService</directoryService>
<!-- The list of supported authentication mechanisms. -->
<saslMechanismHandlers>
<simpleMechanismHandler mech-name="SIMPLE"/>
<cramMd5MechanismHandler mech-name="CRAM-MD5" />
<digestMd5MechanismHandler mech-name="DIGEST-MD5" />
<gssapiMechanismHandler mech-name="GSSAPI" />
<ntlmMechanismHandler mech-name="NTLM" ntlmProviderFqcn="com.foo.Bar"/>
<ntlmMechanismHandler mech-name="GSS-SPNEGO" ntlmProviderFqcn="com.foo.Bar"/>
</saslMechanismHandlers>
<!-- The realms serviced by this SASL host, used by DIGEST-MD5 and GSSAPI. -->
<saslRealms>
<s:value>example.com</s:value>
<s:value>apache.org</s:value>
</saslRealms>
<!-- the collection of extended operation handlers to install -->
<extendedOperationHandlers>
<startTlsHandler/>
<gracefulShutdownHandler/>
<launchDiagnosticUiHandler/>
<!-- The Stored Procedure Extended Operation is not stable yet and it may cause security risks.-->
<!--storedProcedureExtendedOperationHandler/-->
</extendedOperationHandlers>
</ldapServer>
<apacheDS id="apacheDS" ldifDirectory="ldif">
<ldapServer>#ldapServer</ldapServer>
</apacheDS>
<!-- uncomment the below line to start the jetty(v6.1.14) http server
This can be used to provide access to the data present in DIT via http
using a web application
-->
<!--
<httpServer id="httpServer" port="7009" >
<webApps>
<webApp warFile="/path/to/war/file" contextPath="/myApp"/>
</webApps>
</httpServer>
-->
</spring:beans>

22
hadoop-common-project/hadoop-common/src/test/resources/krb5.conf
View File

@ -15,14 +15,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
[libdefaults]
default_realm = APACHE.ORG
udp_preference_limit = 1
extra_addresses = 127.0.0.1
default_realm = EXAMPLE.COM
allow_weak_crypto = true
default_tkt_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
default_tgs_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
[realms]
APACHE.ORG = {
admin_server = localhost:88
kdc = localhost:88
EXAMPLE.COM = {
kdc = localhost:60088
}
[domain_realm]
localhost = APACHE.ORG
.example.com = EXAMPLE.COM
example.com = EXAMPLE.COM
[login]
krb4_convert = true
krb4_get_tickets = false

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -201,6 +201,8 @@ Release 0.23.3 - UNRELEASED
OPTIMIZATIONS
HDFS-3024. Improve performance of stringification in addStoredBlock (todd)
BUG FIXES
HDFS-2481. Unknown protocol: org.apache.hadoop.hdfs.protocol.ClientProtocol.

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java
View File

@ -150,6 +150,14 @@ public class Block implements Writable, Comparable<Block> {
return getBlockName() + "_" + getGenerationStamp();
}
public void appendStringTo(StringBuilder sb) {
sb.append(BLOCK_FILE_PREFIX)
.append(blockId)
.append("_")
.append(getGenerationStamp());
}
/////////////////////////////////////
// Writable
/////////////////////////////////////

40
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hdfs.protocol.Block;
@ -114,13 +115,18 @@ public class BlockInfoUnderConstruction extends BlockInfo {
@Override
public String toString() {
final StringBuilder b = new StringBuilder(getClass().getSimpleName());
b.append("[")
final StringBuilder b = new StringBuilder(50);
appendStringTo(b);
return b.toString();
}
@Override
public void appendStringTo(StringBuilder sb) {
sb.append("ReplicaUnderConstruction[")
.append(expectedLocation)
.append("|")
.append(state)
.append("]");
return b.toString();
}
}
@ -269,11 +275,29 @@ public class BlockInfoUnderConstruction extends BlockInfo {
@Override
public String toString() {
final StringBuilder b = new StringBuilder(super.toString());
b.append("{blockUCState=").append(blockUCState)
.append(", primaryNodeIndex=").append(primaryNodeIndex)
.append(", replicas=").append(replicas)
.append("}");
final StringBuilder b = new StringBuilder(100);
appendStringTo(b);
return b.toString();
}
@Override
public void appendStringTo(StringBuilder sb) {
super.appendStringTo(sb);
appendUCParts(sb);
}
private void appendUCParts(StringBuilder sb) {
sb.append("{blockUCState=").append(blockUCState)
.append(", primaryNodeIndex=").append(primaryNodeIndex)
.append(", replicas=[");
Iterator<ReplicaUnderConstruction> iter = replicas.iterator();
if (iter.hasNext()) {
iter.next().appendStringTo(sb);
while (iter.hasNext()) {
sb.append(", ");
iter.next().appendStringTo(sb);
}
}
sb.append("]}");
}
}

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -1967,9 +1967,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
if (added) {
curReplicaDelta = 1;
if (logEveryBlock) {
NameNode.stateChangeLog.info("BLOCK* addStoredBlock: "
+ "blockMap updated: " + node.getName() + " is added to " +
storedBlock + " size " + storedBlock.getNumBytes());
logAddStoredBlock(storedBlock, node);
}
} else {
curReplicaDelta = 0;
@ -2031,6 +2029,20 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
return storedBlock;
}
private void logAddStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
if (!NameNode.stateChangeLog.isInfoEnabled()) {
return;
}
StringBuilder sb = new StringBuilder(500);
sb.append("BLOCK* addStoredBlock: blockMap updated: ")
.append(node.getName())
.append(" is added to ");
storedBlock.appendStringTo(sb);
sb.append(" size " )
.append(storedBlock.getNumBytes());
NameNode.stateChangeLog.info(sb);
}
/**
* Invalidate corrupt replicas.
* <p>

6
hadoop-mapreduce-project/CHANGES.txt
View File

@ -194,6 +194,12 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3913. RM application webpage is unresponsive after 2000 jobs
(Jason Lowe via tgraves)
MAPREDUCE-3922. Fixed build to not compile 32bit container-executor binary
by default on all platforms. (Hitesh Shah via vinodkv)
MAPREDUCE-3790 Broken pipe on streaming job can lead to truncated output for
a successful job (Jason Lowe via bobby)
Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES

3
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
View File

@ -28,6 +28,7 @@
<!-- Basedir eeded for generating FindBugs warnings using parent pom -->
<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
<container-executor.conf.dir>/etc/hadoop</container-executor.conf.dir>
<container-executor.additional_cflags></container-executor.additional_cflags>
</properties>
<dependencies>
@ -74,7 +75,7 @@
<configureEnvironment>
<property>
<name>CFLAGS</name>
<value>-DHADOOP_CONF_DIR=${container-executor.conf.dir} -m32</value>
<value>-DHADOOP_CONF_DIR=${container-executor.conf.dir} ${container-executor.additional_cflags}</value>
</property>
</configureEnvironment>
<configureWorkDir>${project.build.directory}/native/container-executor</configureWorkDir>

6
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
View File

@ -521,11 +521,15 @@ public abstract class PipeMapRed {
LOG.info("mapRedFinished");
return;
}
try {
if (clientOut_ != null) {
try {
clientOut_.flush();
clientOut_.close();
} catch (IOException io) {
LOG.warn(io);
}
}
try {
waitOutputThreads();
} catch (IOException io) {
LOG.warn(io);

38
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/OutputOnlyApp.java Normal file
View File

@ -0,0 +1,38 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.streaming;
import java.io.IOException;
/**
* An application that outputs a specified number of lines
* without consuming any input.
*/
public class OutputOnlyApp {
public static void main(String[] args) throws IOException {
if (args.length < 1) {
System.err.println("Usage: OutputOnlyApp NUMRECORDS");
return;
}
int numRecords = Integer.parseInt(args[0]);
while (numRecords-- > 0) {
System.out.println("key\tvalue");
}
}
}

107
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java Normal file
View File

@ -0,0 +1,107 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.streaming;
import static org.junit.Assert.*;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.Test;
public class TestUnconsumedInput {
protected final int EXPECTED_OUTPUT_SIZE = 10000;
protected File INPUT_FILE = new File("stream_uncinput_input.txt");
protected File OUTPUT_DIR = new File("stream_uncinput_out");
// map parses input lines and generates count entries for each word.
protected String input = "roses.are.red\nviolets.are.blue\nbunnies.are.pink\n";
protected String map = UtilTest.makeJavaCommand(OutputOnlyApp.class,
new String[]{Integer.toString(EXPECTED_OUTPUT_SIZE)});
private StreamJob job;
public TestUnconsumedInput() throws IOException
{
UtilTest utilTest = new UtilTest(getClass().getName());
utilTest.checkUserDir();
utilTest.redirectIfAntJunit();
}
protected void createInput() throws IOException
{
DataOutputStream out = new DataOutputStream(
new FileOutputStream(INPUT_FILE.getAbsoluteFile()));
for (int i=0; i<10000; ++i) {
out.write(input.getBytes("UTF-8"));
}
out.close();
}
protected String[] genArgs() {
return new String[] {
"-input", INPUT_FILE.getAbsolutePath(),
"-output", OUTPUT_DIR.getAbsolutePath(),
"-mapper", map,
"-reducer", "org.apache.hadoop.mapred.lib.IdentityReducer",
"-numReduceTasks", "0",
"-jobconf", "mapreduce.task.files.preserve.failedtasks=true",
"-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
};
}
@Test
public void testUnconsumedInput() throws Exception
{
String outFileName = "part-00000";
File outFile = null;
try {
try {
FileUtil.fullyDelete(OUTPUT_DIR.getAbsoluteFile());
} catch (Exception e) {
}
createInput();
// setup config to ignore unconsumed input
Configuration conf = new Configuration();
conf.set("stream.minRecWrittenToEnableSkip_", "0");
job = new StreamJob();
job.setConf(conf);
int exitCode = job.run(genArgs());
assertEquals("Job failed", 0, exitCode);
outFile = new File(OUTPUT_DIR, outFileName).getAbsoluteFile();
String output = StreamUtil.slurp(outFile);
assertEquals("Output was truncated", EXPECTED_OUTPUT_SIZE,
StringUtils.countMatches(output, "\t"));
} finally {
INPUT_FILE.delete();
FileUtil.fullyDelete(OUTPUT_DIR.getAbsoluteFile());
}
}
}