Merge branch 'trunk' into HDFS-6581

This commit is contained in:
arp 2014-09-08 10:37:26 -07:00
commit 6e5f780d1d
14 changed files with 171 additions and 39 deletions

View File

@ -768,6 +768,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
determine if in proxyuser mode or not. (tucu) determine if in proxyuser mode or not. (tucu)
HADOOP-11073. Credential Provider related Unit Tests Failure on Windows.
(Xiaoyu Yao via cnauroth)
Release 2.5.1 - UNRELEASED Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -55,18 +55,18 @@ public class TestKeyProviderFactory {
@Test @Test
public void testFactory() throws Exception { public void testFactory() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
final String userUri = UserProvider.SCHEME_NAME + ":///";
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String jksUri = JavaKeyStoreProvider.SCHEME_NAME +
"://file" + jksPath.toUri().toString();
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
UserProvider.SCHEME_NAME + ":///," + userUri + "," + jksUri);
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
List<KeyProvider> providers = KeyProviderFactory.getProviders(conf); List<KeyProvider> providers = KeyProviderFactory.getProviders(conf);
assertEquals(2, providers.size()); assertEquals(2, providers.size());
assertEquals(UserProvider.class, providers.get(0).getClass()); assertEquals(UserProvider.class, providers.get(0).getClass());
assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass()); assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass());
assertEquals(UserProvider.SCHEME_NAME + assertEquals(userUri, providers.get(0).toString());
":///", providers.get(0).toString()); assertEquals(jksUri, providers.get(1).toString());
assertEquals(JavaKeyStoreProvider.SCHEME_NAME +
"://file" + tmpDir + "/test.jks",
providers.get(1).toString());
} }
@Test @Test
@ -207,8 +207,9 @@ public class TestKeyProviderFactory {
@Test @Test
public void testJksProvider() throws Exception { public void testJksProvider() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String ourUrl = final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"; JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(tmpDir, "test.jks"); File file = new File(tmpDir, "test.jks");
file.delete(); file.delete();
@ -317,8 +318,9 @@ public class TestKeyProviderFactory {
@Test @Test
public void testJksProviderPasswordViaConfig() throws Exception { public void testJksProviderPasswordViaConfig() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String ourUrl = final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"; JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(tmpDir, "test.jks"); File file = new File(tmpDir, "test.jks");
file.delete(); file.delete();
try { try {
@ -360,8 +362,8 @@ public class TestKeyProviderFactory {
@Test @Test
public void testGetProviderViaURI() throws Exception { public void testGetProviderViaURI() throws Exception {
Configuration conf = new Configuration(false); Configuration conf = new Configuration(false);
URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + final Path jksPath = new Path(tmpDir.toString(), "test.jks");
"/test.jks"); URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
KeyProvider kp = KeyProviderFactory.get(uri, conf); KeyProvider kp = KeyProviderFactory.get(uri, conf);
Assert.assertNotNull(kp); Assert.assertNotNull(kp);
Assert.assertEquals(JavaKeyStoreProvider.class, kp.getClass()); Assert.assertEquals(JavaKeyStoreProvider.class, kp.getClass());

View File

@ -40,6 +40,7 @@ import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult; import javax.naming.directory.SearchResult;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@ -165,8 +166,9 @@ public class TestLdapGroupsMapping {
File testDir = new File(System.getProperty("test.build.data", File testDir = new File(System.getProperty("test.build.data",
"target/test-dir")); "target/test-dir"));
Configuration conf = new Configuration(); Configuration conf = new Configuration();
final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl = final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks"); File file = new File(testDir, "test.jks");
file.delete(); file.delete();

View File

@ -52,19 +52,19 @@ public class TestCredentialProviderFactory {
@Test @Test
public void testFactory() throws Exception { public void testFactory() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
final String userUri = UserProvider.SCHEME_NAME + ":///";
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String jksUri = JavaKeyStoreProvider.SCHEME_NAME +
"://file" + jksPath.toUri();
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
UserProvider.SCHEME_NAME + ":///," + userUri + "," + jksUri);
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
List<CredentialProvider> providers = List<CredentialProvider> providers =
CredentialProviderFactory.getProviders(conf); CredentialProviderFactory.getProviders(conf);
assertEquals(2, providers.size()); assertEquals(2, providers.size());
assertEquals(UserProvider.class, providers.get(0).getClass()); assertEquals(UserProvider.class, providers.get(0).getClass());
assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass()); assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass());
assertEquals(UserProvider.SCHEME_NAME + assertEquals(userUri, providers.get(0).toString());
":///", providers.get(0).toString()); assertEquals(jksUri, providers.get(1).toString());
assertEquals(JavaKeyStoreProvider.SCHEME_NAME +
"://file" + tmpDir + "/test.jks",
providers.get(1).toString());
} }
@Test @Test
@ -188,8 +188,9 @@ public class TestCredentialProviderFactory {
@Test @Test
public void testJksProvider() throws Exception { public void testJksProvider() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
final String ourUrl = final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"; JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(tmpDir, "test.jks"); File file = new File(tmpDir, "test.jks");
file.delete(); file.delete();

View File

@ -19,6 +19,7 @@
package org.apache.hadoop.security.ssl; package org.apache.hadoop.security.ssl;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@ -392,8 +393,9 @@ public class KeyStoreTestUtil {
"target/test-dir")); "target/test-dir"));
Configuration conf = new Configuration(); Configuration conf = new Configuration();
final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl = final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks"); File file = new File(testDir, "test.jks");
file.delete(); file.delete();

View File

@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@ -305,8 +306,9 @@ public class TestSSLFactory {
if (useCredProvider) { if (useCredProvider) {
File testDir = new File(System.getProperty("test.build.data", File testDir = new File(System.getProperty("test.build.data",
"target/test-dir")); "target/test-dir"));
final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl = final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
sslConf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); sslConf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
} }
} else { } else {

View File

@ -184,6 +184,15 @@ Release 2.6.0 - UNRELEASED
YARN-2508. Cross Origin configuration parameters prefix are not honored YARN-2508. Cross Origin configuration parameters prefix are not honored
(Mit Desai via jeagles) (Mit Desai via jeagles)
YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
(Jonathan Eagles via zjshen)
YARN-2507. Documented CrossOriginFilter configurations for the timeline
server. (Jonathan Eagles via zjshen)
YARN-2515. Updated ConverterUtils#toContainerId to parse epoch.
(Tsuyoshi OZAWA via jianhe)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.api.records; package org.apache.hadoop.yarn.api.records;
import java.text.NumberFormat; import com.google.common.base.Splitter;
import java.text.NumberFormat;
import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Stable;
@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.util.Records;
@Public @Public
@Stable @Stable
public abstract class ContainerId implements Comparable<ContainerId>{ public abstract class ContainerId implements Comparable<ContainerId>{
private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
private static final String CONTAINER_PREFIX = "container";
@Private @Private
@Unstable @Unstable
@ -163,5 +167,38 @@ public abstract class ContainerId implements Comparable<ContainerId>{
return sb.toString(); return sb.toString();
} }
@Public
@Unstable
public static ContainerId fromString(String containerIdStr) {
Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
if (!it.next().equals(CONTAINER_PREFIX)) {
throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ containerIdStr);
}
try {
ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
int id = Integer.parseInt(it.next());
int epoch = 0;
if (it.hasNext()) {
epoch = Integer.parseInt(it.next());
}
int cid = (epoch << 22) | id;
ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
return containerId;
} catch (NumberFormatException n) {
throw new IllegalArgumentException("Invalid ContainerId: "
+ containerIdStr, n);
}
}
private static ApplicationAttemptId toApplicationAttemptId(
Iterator<String> it) throws NumberFormatException {
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
Integer.parseInt(it.next()));
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
return appAttemptId;
}
protected abstract void build(); protected abstract void build();
} }

View File

@ -168,20 +168,7 @@ public class ConverterUtils {
} }
public static ContainerId toContainerId(String containerIdStr) { public static ContainerId toContainerId(String containerIdStr) {
Iterator<String> it = _split(containerIdStr).iterator(); return ContainerId.fromString(containerIdStr);
if (!it.next().equals(CONTAINER_PREFIX)) {
throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ containerIdStr);
}
try {
ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
ContainerId containerId =
ContainerId.newInstance(appAttemptID, Integer.parseInt(it.next()));
return containerId;
} catch (NumberFormatException n) {
throw new IllegalArgumentException("Invalid ContainerId: "
+ containerIdStr, n);
}
} }
public static ApplicationAttemptId toApplicationAttemptId( public static ApplicationAttemptId toApplicationAttemptId(

View File

@ -54,10 +54,14 @@ public class TestContainerId {
long ts = System.currentTimeMillis(); long ts = System.currentTimeMillis();
ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811); ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811);
Assert.assertEquals("container_10_0001_01_000001", c1.toString()); Assert.assertEquals("container_10_0001_01_000001", c1.toString());
Assert.assertEquals(c1,
ContainerId.fromString("container_10_0001_01_000001"));
Assert.assertEquals(479987, 0x003fffff & c6.getId()); Assert.assertEquals(479987, 0x003fffff & c6.getId());
Assert.assertEquals(6, c6.getId() >> 22); Assert.assertEquals(6, c6.getId() >> 22);
Assert.assertEquals("container_" + ts + "_36473_4365472_479987_06", Assert.assertEquals("container_" + ts + "_36473_4365472_479987_06",
c6.toString()); c6.toString());
Assert.assertEquals(c6,
ContainerId.fromString("container_" + ts + "_36473_4365472_479987_06"));
} }
public static ContainerId newContainerId(int appId, int appAttemptId, public static ContainerId newContainerId(int appId, int appAttemptId,

View File

@ -55,6 +55,15 @@ public class TestConverterUtils {
assertEquals(gen, id); assertEquals(gen, id);
} }
@Test
public void testContainerIdWithEpoch() throws URISyntaxException {
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
String cid = ConverterUtils.toString(id);
assertEquals("container_0_0000_00_479987_06", cid);
ContainerId gen = ConverterUtils.toContainerId(cid);
assertEquals(gen.toString(), id.toString());
}
@Test @Test
public void testContainerIdNull() throws URISyntaxException { public void testContainerIdNull() throws URISyntaxException {
assertNull(ConverterUtils.toString((ContainerId)null)); assertNull(ConverterUtils.toString((ContainerId)null));

View File

@ -24,6 +24,8 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.Filter; import javax.servlet.Filter;
import javax.servlet.FilterChain; import javax.servlet.FilterChain;
@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter {
@VisibleForTesting @VisibleForTesting
boolean isOriginAllowed(String origin) { boolean isOriginAllowed(String origin) {
return allowAllOrigins || allowedOrigins.contains(origin); if (allowAllOrigins) {
return true;
}
for (String allowedOrigin : allowedOrigins) {
if (allowedOrigin.contains("*")) {
String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(origin);
if (m.matches()) {
return true;
}
} else if (allowedOrigin.equals(origin)) {
return true;
}
}
return false;
} }
private boolean areHeadersAllowed(String accessControlRequestHeaders) { private boolean areHeadersAllowed(String accessControlRequestHeaders) {

View File

@ -77,7 +77,26 @@ public class TestCrossOriginFilter {
// Object under test // Object under test
CrossOriginFilter filter = new CrossOriginFilter(); CrossOriginFilter filter = new CrossOriginFilter();
filter.init(filterConfig); filter.init(filterConfig);
Assert.assertTrue(filter.isOriginAllowed("example.org")); Assert.assertTrue(filter.isOriginAllowed("example.com"));
}
@Test
public void testPatternMatchingOrigins() throws ServletException, IOException {
// Setup the configuration settings of the server
Map<String, String> conf = new HashMap<String, String>();
conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "*.example.com");
FilterConfig filterConfig = new FilterConfigTest(conf);
// Object under test
CrossOriginFilter filter = new CrossOriginFilter();
filter.init(filterConfig);
// match multiple sub-domains
Assert.assertFalse(filter.isOriginAllowed("example.com"));
Assert.assertFalse(filter.isOriginAllowed("foo:example.com"));
Assert.assertTrue(filter.isOriginAllowed("foo.example.com"));
Assert.assertTrue(filter.isOriginAllowed("foo.bar.example.com"));
} }
@Test @Test

View File

@ -102,6 +102,43 @@ YARN Timeline Server
<name>yarn.timeline-service.handler-thread-count</name> <name>yarn.timeline-service.handler-thread-count</name>
<value>10</value> <value>10</value>
</property> </property>
<property>
<description>Enables cross-origin support (CORS) for web services where
cross-origin web response headers are needed. For example, javascript making
a web services request to the timeline server.</description>
<name>yarn.timeline-service.http-cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>Comma separated list of origins that are allowed for web
services needing cross-origin (CORS) support. Wildcards (*) and patterns
allowed</description>
<name>yarn.timeline-service.http-cross-origin.allowed-origins</name>
<value>*</value>
</property>
<property>
<description>Comma separated list of methods that are allowed for web
services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.allowed-methods</name>
<value>GET,POST,HEAD</value>
</property>
<property>
<description>Comma separated list of headers that are allowed for web
services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.allowed-headers</name>
<value>X-Requested-With,Content-Type,Accept,Origin</value>
</property>
<property>
<description>The number of seconds a pre-flighted request can be cached
for web services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.max-age</name>
<value>1800</value>
</property>
+---+ +---+
* Generic-data related Configuration * Generic-data related Configuration