Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'trunk' into HDFS-6581

This commit is contained in:
arp 2014-09-08 10:37:26 -07:00
parent ccdf0054a3 302d9a0a0d
commit 6e5f780d1d
14 changed files with 171 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -768,6 +768,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
determine if in proxyuser mode or not. (tucu)
HADOOP-11073. Credential Provider related Unit Tests Failure on Windows.
(Xiaoyu Yao via cnauroth)
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES

24
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
View File

@ -55,18 +55,18 @@ public void setup() {
@Test
public void testFactory() throws Exception {
Configuration conf = new Configuration();
final String userUri = UserProvider.SCHEME_NAME + ":///";
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String jksUri = JavaKeyStoreProvider.SCHEME_NAME +
"://file" + jksPath.toUri().toString();
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
UserProvider.SCHEME_NAME + ":///," +
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
userUri + "," + jksUri);
List<KeyProvider> providers = KeyProviderFactory.getProviders(conf);
assertEquals(2, providers.size());
assertEquals(UserProvider.class, providers.get(0).getClass());
assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass());
assertEquals(UserProvider.SCHEME_NAME +
":///", providers.get(0).toString());
assertEquals(JavaKeyStoreProvider.SCHEME_NAME +
"://file" + tmpDir + "/test.jks",
providers.get(1).toString());
assertEquals(userUri, providers.get(0).toString());
assertEquals(jksUri, providers.get(1).toString());
}
@Test
@ -207,8 +207,9 @@ public void testUserProvider() throws Exception {
@Test
public void testJksProvider() throws Exception {
Configuration conf = new Configuration();
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(tmpDir, "test.jks");
file.delete();
@ -317,8 +318,9 @@ public void checkPermissionRetention(Configuration conf, String ourUrl, Path pat
@Test
public void testJksProviderPasswordViaConfig() throws Exception {
Configuration conf = new Configuration();
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(tmpDir, "test.jks");
file.delete();
try {
@ -360,8 +362,8 @@ public void testJksProviderPasswordViaConfig() throws Exception {
@Test
public void testGetProviderViaURI() throws Exception {
Configuration conf = new Configuration(false);
URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir +
"/test.jks");
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
KeyProvider kp = KeyProviderFactory.get(uri, conf);
Assert.assertNotNull(kp);
Assert.assertEquals(JavaKeyStoreProvider.class, kp.getClass());

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
View File

@ -40,6 +40,7 @@
import javax.naming.directory.SearchResult;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@ -165,8 +166,9 @@ public void testConfGetPassword() throws Exception {
File testDir = new File(System.getProperty("test.build.data",
"target/test-dir"));
Configuration conf = new Configuration();
final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks");
file.delete();

17
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
View File

@ -52,19 +52,19 @@ public class TestCredentialProviderFactory {
@Test
public void testFactory() throws Exception {
Configuration conf = new Configuration();
final String userUri = UserProvider.SCHEME_NAME + ":///";
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String jksUri = JavaKeyStoreProvider.SCHEME_NAME +
"://file" + jksPath.toUri();
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
UserProvider.SCHEME_NAME + ":///," +
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
userUri + "," + jksUri);
List<CredentialProvider> providers =
CredentialProviderFactory.getProviders(conf);
assertEquals(2, providers.size());
assertEquals(UserProvider.class, providers.get(0).getClass());
assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass());
assertEquals(UserProvider.SCHEME_NAME +
":///", providers.get(0).toString());
assertEquals(JavaKeyStoreProvider.SCHEME_NAME +
"://file" + tmpDir + "/test.jks",
providers.get(1).toString());
assertEquals(userUri, providers.get(0).toString());
assertEquals(jksUri, providers.get(1).toString());
}
@Test
@ -188,8 +188,9 @@ public void testUserProvider() throws Exception {
@Test
public void testJksProvider() throws Exception {
Configuration conf = new Configuration();
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(tmpDir, "test.jks");
file.delete();

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.security.ssl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@ -392,8 +393,9 @@ public static void provisionPasswordsToCredentialProvider() throws Exception {
"target/test-dir"));
Configuration conf = new Configuration();
final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks");
file.delete();

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@ -305,8 +306,9 @@ private void checkSSLFactoryInitWithPasswords(SSLFactory.Mode mode,
if (useCredProvider) {
File testDir = new File(System.getProperty("test.build.data",
"target/test-dir"));
final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
sslConf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
}
} else {

9
hadoop-yarn-project/CHANGES.txt
View File

@ -184,6 +184,15 @@ Release 2.6.0 - UNRELEASED
YARN-2508. Cross Origin configuration parameters prefix are not honored
(Mit Desai via jeagles)
YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
(Jonathan Eagles via zjshen)
YARN-2507. Documented CrossOriginFilter configurations for the timeline
server. (Jonathan Eagles via zjshen)
YARN-2515. Updated ConverterUtils#toContainerId to parse epoch.
(Tsuyoshi OZAWA via jianhe)
OPTIMIZATIONS
BUG FIXES

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
View File

@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.api.records;
import java.text.NumberFormat;
import com.google.common.base.Splitter;
import java.text.NumberFormat;
import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
@ -33,6 +35,8 @@
@Public
@Stable
public abstract class ContainerId implements Comparable<ContainerId>{
private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
private static final String CONTAINER_PREFIX = "container";
@Private
@Unstable
@ -163,5 +167,38 @@ public String toString() {
return sb.toString();
}
@Public
@Unstable
public static ContainerId fromString(String containerIdStr) {
Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
if (!it.next().equals(CONTAINER_PREFIX)) {
throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ containerIdStr);
}
try {
ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
int id = Integer.parseInt(it.next());
int epoch = 0;
if (it.hasNext()) {
epoch = Integer.parseInt(it.next());
}
int cid = (epoch << 22) | id;
ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
return containerId;
} catch (NumberFormatException n) {
throw new IllegalArgumentException("Invalid ContainerId: "
+ containerIdStr, n);
}
}
private static ApplicationAttemptId toApplicationAttemptId(
Iterator<String> it) throws NumberFormatException {
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
Integer.parseInt(it.next()));
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
return appAttemptId;
}
protected abstract void build();
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
View File

@ -168,20 +168,7 @@ public static NodeId toNodeId(String nodeIdStr) {
}
public static ContainerId toContainerId(String containerIdStr) {
Iterator<String> it = _split(containerIdStr).iterator();
if (!it.next().equals(CONTAINER_PREFIX)) {
throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ containerIdStr);
}
try {
ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
ContainerId containerId =
ContainerId.newInstance(appAttemptID, Integer.parseInt(it.next()));
return containerId;
} catch (NumberFormatException n) {
throw new IllegalArgumentException("Invalid ContainerId: "
+ containerIdStr, n);
}
return ContainerId.fromString(containerIdStr);
}
public static ApplicationAttemptId toApplicationAttemptId(

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
View File

@ -54,10 +54,14 @@ public void testContainerId() {
long ts = System.currentTimeMillis();
ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811);
Assert.assertEquals("container_10_0001_01_000001", c1.toString());
Assert.assertEquals(c1,
ContainerId.fromString("container_10_0001_01_000001"));
Assert.assertEquals(479987, 0x003fffff & c6.getId());
Assert.assertEquals(6, c6.getId() >> 22);
Assert.assertEquals("container_" + ts + "_36473_4365472_479987_06",
c6.toString());
Assert.assertEquals(c6,
ContainerId.fromString("container_" + ts + "_36473_4365472_479987_06"));
}
public static ContainerId newContainerId(int appId, int appAttemptId,

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
View File

@ -55,6 +55,15 @@ public void testContainerId() throws URISyntaxException {
assertEquals(gen, id);
}
@Test
public void testContainerIdWithEpoch() throws URISyntaxException {
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
String cid = ConverterUtils.toString(id);
assertEquals("container_0_0000_00_479987_06", cid);
ContainerId gen = ConverterUtils.toContainerId(cid);
assertEquals(gen.toString(), id.toString());
}
@Test
public void testContainerIdNull() throws URISyntaxException {
assertNull(ConverterUtils.toString((ContainerId)null));

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
View File

@ -24,6 +24,8 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@ -204,7 +206,23 @@ static boolean isCrossOrigin(String origin) {
@VisibleForTesting
boolean isOriginAllowed(String origin) {
return allowAllOrigins || allowedOrigins.contains(origin);
if (allowAllOrigins) {
return true;
}
for (String allowedOrigin : allowedOrigins) {
if (allowedOrigin.contains("*")) {
String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(origin);
if (m.matches()) {
return true;
}
} else if (allowedOrigin.equals(origin)) {
return true;
}
}
return false;
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
View File

@ -77,7 +77,26 @@ public void testAllowAllOrigins() throws ServletException, IOException {
// Object under test
CrossOriginFilter filter = new CrossOriginFilter();
filter.init(filterConfig);
Assert.assertTrue(filter.isOriginAllowed("example.org"));
Assert.assertTrue(filter.isOriginAllowed("example.com"));
}
@Test
public void testPatternMatchingOrigins() throws ServletException, IOException {
// Setup the configuration settings of the server
Map<String, String> conf = new HashMap<String, String>();
conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "*.example.com");
FilterConfig filterConfig = new FilterConfigTest(conf);
// Object under test
CrossOriginFilter filter = new CrossOriginFilter();
filter.init(filterConfig);
// match multiple sub-domains
Assert.assertFalse(filter.isOriginAllowed("example.com"));
Assert.assertFalse(filter.isOriginAllowed("foo:example.com"));
Assert.assertTrue(filter.isOriginAllowed("foo.example.com"));
Assert.assertTrue(filter.isOriginAllowed("foo.bar.example.com"));
}
@Test

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
View File

@ -102,6 +102,43 @@ YARN Timeline Server
<name>yarn.timeline-service.handler-thread-count</name>
<value>10</value>
</property>
<property>
<description>Enables cross-origin support (CORS) for web services where
cross-origin web response headers are needed. For example, javascript making
a web services request to the timeline server.</description>
<name>yarn.timeline-service.http-cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>Comma separated list of origins that are allowed for web
services needing cross-origin (CORS) support. Wildcards (*) and patterns
allowed</description>
<name>yarn.timeline-service.http-cross-origin.allowed-origins</name>
<value>*</value>
</property>
<property>
<description>Comma separated list of methods that are allowed for web
services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.allowed-methods</name>
<value>GET,POST,HEAD</value>
</property>
<property>
<description>Comma separated list of headers that are allowed for web
services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.allowed-headers</name>
<value>X-Requested-With,Content-Type,Accept,Origin</value>
</property>
<property>
<description>The number of seconds a pre-flighted request can be cached
for web services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.max-age</name>
<value>1800</value>
</property>
+---+
* Generic-data related Configuration