diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c77fddc7899..fe011fd0395 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -768,6 +768,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu) + HADOOP-11073. Credential Provider related Unit Tests Failure on Windows. + (Xiaoyu Yao via cnauroth) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java index 8c4c7b328bf..8fb661d452d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java @@ -55,18 +55,18 @@ public class TestKeyProviderFactory { @Test public void testFactory() throws Exception { Configuration conf = new Configuration(); + final String userUri = UserProvider.SCHEME_NAME + ":///"; + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); + final String jksUri = JavaKeyStoreProvider.SCHEME_NAME + + "://file" + jksPath.toUri().toString(); conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, - UserProvider.SCHEME_NAME + ":///," + - JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"); + userUri + "," + jksUri); List providers = KeyProviderFactory.getProviders(conf); assertEquals(2, providers.size()); assertEquals(UserProvider.class, providers.get(0).getClass()); assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass()); - assertEquals(UserProvider.SCHEME_NAME + - ":///", providers.get(0).toString()); - assertEquals(JavaKeyStoreProvider.SCHEME_NAME + - "://file" + tmpDir + "/test.jks", - providers.get(1).toString()); + assertEquals(userUri, providers.get(0).toString()); + assertEquals(jksUri, providers.get(1).toString()); } @Test @@ -207,8 +207,9 @@ public class TestKeyProviderFactory { @Test public void testJksProvider() throws Exception { Configuration conf = new Configuration(); + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); final String ourUrl = - JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"; + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); File file = new File(tmpDir, "test.jks"); file.delete(); @@ -317,8 +318,9 @@ public class TestKeyProviderFactory { @Test public void testJksProviderPasswordViaConfig() throws Exception { Configuration conf = new Configuration(); + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); final String ourUrl = - JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"; + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); File file = new File(tmpDir, "test.jks"); file.delete(); try { @@ -360,8 +362,8 @@ public class TestKeyProviderFactory { @Test public void testGetProviderViaURI() throws Exception { Configuration conf = new Configuration(false); - URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + - "/test.jks"); + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); + URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()); KeyProvider kp = KeyProviderFactory.get(uri, conf); Assert.assertNotNull(kp); Assert.assertEquals(JavaKeyStoreProvider.class, kp.getClass()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java index 95cbc0ae201..a021a8afd4a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java @@ -40,6 +40,7 @@ import javax.naming.directory.SearchControls; import javax.naming.directory.SearchResult; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; @@ -165,8 +166,9 @@ public class TestLdapGroupsMapping { File testDir = new File(System.getProperty("test.build.data", "target/test-dir")); Configuration conf = new Configuration(); + final Path jksPath = new Path(testDir.toString(), "test.jks"); final String ourUrl = - JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); File file = new File(testDir, "test.jks"); file.delete(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java index 3d8794fb822..1eb1e8be98e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java @@ -52,19 +52,19 @@ public class TestCredentialProviderFactory { @Test public void testFactory() throws Exception { Configuration conf = new Configuration(); + final String userUri = UserProvider.SCHEME_NAME + ":///"; + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); + final String jksUri = JavaKeyStoreProvider.SCHEME_NAME + + "://file" + jksPath.toUri(); conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, - UserProvider.SCHEME_NAME + ":///," + - JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"); + userUri + "," + jksUri); List providers = CredentialProviderFactory.getProviders(conf); assertEquals(2, providers.size()); assertEquals(UserProvider.class, providers.get(0).getClass()); assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass()); - assertEquals(UserProvider.SCHEME_NAME + - ":///", providers.get(0).toString()); - assertEquals(JavaKeyStoreProvider.SCHEME_NAME + - "://file" + tmpDir + "/test.jks", - providers.get(1).toString()); + assertEquals(userUri, providers.get(0).toString()); + assertEquals(jksUri, providers.get(1).toString()); } @Test @@ -188,8 +188,9 @@ public class TestCredentialProviderFactory { @Test public void testJksProvider() throws Exception { Configuration conf = new Configuration(); + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); final String ourUrl = - JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"; + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); File file = new File(tmpDir, "test.jks"); file.delete(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java index 5f72f9e95eb..b2a839ca2a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java @@ -19,6 +19,7 @@ package org.apache.hadoop.security.ssl; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; @@ -392,8 +393,9 @@ public class KeyStoreTestUtil { "target/test-dir")); Configuration conf = new Configuration(); + final Path jksPath = new Path(testDir.toString(), "test.jks"); final String ourUrl = - JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); File file = new File(testDir, "test.jks"); file.delete(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java index 3c428fea282..004888c7b71 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; @@ -305,8 +306,9 @@ public class TestSSLFactory { if (useCredProvider) { File testDir = new File(System.getProperty("test.build.data", "target/test-dir")); + final Path jksPath = new Path(testDir.toString(), "test.jks"); final String ourUrl = - JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); sslConf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); } } else { diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index beafc225188..d54fcd6fef1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -184,6 +184,15 @@ Release 2.6.0 - UNRELEASED YARN-2508. Cross Origin configuration parameters prefix are not honored (Mit Desai via jeagles) + YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. + (Jonathan Eagles via zjshen) + + YARN-2507. Documented CrossOriginFilter configurations for the timeline + server. (Jonathan Eagles via zjshen) + + YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. + (Tsuyoshi OZAWA via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index fc7f40488ec..321052b062b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.api.records; -import java.text.NumberFormat; +import com.google.common.base.Splitter; +import java.text.NumberFormat; +import java.util.Iterator; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; @@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.util.Records; @Public @Stable public abstract class ContainerId implements Comparable{ + private static final Splitter _SPLITTER = Splitter.on('_').trimResults(); + private static final String CONTAINER_PREFIX = "container"; @Private @Unstable @@ -163,5 +167,38 @@ public abstract class ContainerId implements Comparable{ return sb.toString(); } + @Public + @Unstable + public static ContainerId fromString(String containerIdStr) { + Iterator it = _SPLITTER.split(containerIdStr).iterator(); + if (!it.next().equals(CONTAINER_PREFIX)) { + throw new IllegalArgumentException("Invalid ContainerId prefix: " + + containerIdStr); + } + try { + ApplicationAttemptId appAttemptID = toApplicationAttemptId(it); + int id = Integer.parseInt(it.next()); + int epoch = 0; + if (it.hasNext()) { + epoch = Integer.parseInt(it.next()); + } + int cid = (epoch << 22) | id; + ContainerId containerId = ContainerId.newInstance(appAttemptID, cid); + return containerId; + } catch (NumberFormatException n) { + throw new IllegalArgumentException("Invalid ContainerId: " + + containerIdStr, n); + } + } + + private static ApplicationAttemptId toApplicationAttemptId( + Iterator it) throws NumberFormatException { + ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), + Integer.parseInt(it.next())); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next())); + return appAttemptId; + } + protected abstract void build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index f731af95fc5..27f7bc107e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -168,20 +168,7 @@ public class ConverterUtils { } public static ContainerId toContainerId(String containerIdStr) { - Iterator it = _split(containerIdStr).iterator(); - if (!it.next().equals(CONTAINER_PREFIX)) { - throw new IllegalArgumentException("Invalid ContainerId prefix: " - + containerIdStr); - } - try { - ApplicationAttemptId appAttemptID = toApplicationAttemptId(it); - ContainerId containerId = - ContainerId.newInstance(appAttemptID, Integer.parseInt(it.next())); - return containerId; - } catch (NumberFormatException n) { - throw new IllegalArgumentException("Invalid ContainerId: " - + containerIdStr, n); - } + return ContainerId.fromString(containerIdStr); } public static ApplicationAttemptId toApplicationAttemptId( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java index b23d0ed3fbc..8baf244dda2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java @@ -54,10 +54,14 @@ public class TestContainerId { long ts = System.currentTimeMillis(); ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811); Assert.assertEquals("container_10_0001_01_000001", c1.toString()); + Assert.assertEquals(c1, + ContainerId.fromString("container_10_0001_01_000001")); Assert.assertEquals(479987, 0x003fffff & c6.getId()); Assert.assertEquals(6, c6.getId() >> 22); Assert.assertEquals("container_" + ts + "_36473_4365472_479987_06", c6.toString()); + Assert.assertEquals(c6, + ContainerId.fromString("container_" + ts + "_36473_4365472_479987_06")); } public static ContainerId newContainerId(int appId, int appAttemptId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java index 21af4555e37..3f4147c4189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java @@ -55,6 +55,15 @@ public class TestConverterUtils { assertEquals(gen, id); } + @Test + public void testContainerIdWithEpoch() throws URISyntaxException { + ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811); + String cid = ConverterUtils.toString(id); + assertEquals("container_0_0000_00_479987_06", cid); + ContainerId gen = ConverterUtils.toContainerId(cid); + assertEquals(gen.toString(), id.toString()); + } + @Test public void testContainerIdNull() throws URISyntaxException { assertNull(ConverterUtils.toString((ContainerId)null)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java index d71175f96dc..5a0703d4047 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java @@ -24,6 +24,8 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter { @VisibleForTesting boolean isOriginAllowed(String origin) { - return allowAllOrigins || allowedOrigins.contains(origin); + if (allowAllOrigins) { + return true; + } + + for (String allowedOrigin : allowedOrigins) { + if (allowedOrigin.contains("*")) { + String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*"); + Pattern p = Pattern.compile(regex); + Matcher m = p.matcher(origin); + if (m.matches()) { + return true; + } + } else if (allowedOrigin.equals(origin)) { + return true; + } + } + return false; } private boolean areHeadersAllowed(String accessControlRequestHeaders) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java index f666c21b1ef..ccc9bbfde56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java @@ -77,7 +77,26 @@ public class TestCrossOriginFilter { // Object under test CrossOriginFilter filter = new CrossOriginFilter(); filter.init(filterConfig); - Assert.assertTrue(filter.isOriginAllowed("example.org")); + Assert.assertTrue(filter.isOriginAllowed("example.com")); + } + + @Test + public void testPatternMatchingOrigins() throws ServletException, IOException { + + // Setup the configuration settings of the server + Map conf = new HashMap(); + conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "*.example.com"); + FilterConfig filterConfig = new FilterConfigTest(conf); + + // Object under test + CrossOriginFilter filter = new CrossOriginFilter(); + filter.init(filterConfig); + + // match multiple sub-domains + Assert.assertFalse(filter.isOriginAllowed("example.com")); + Assert.assertFalse(filter.isOriginAllowed("foo:example.com")); + Assert.assertTrue(filter.isOriginAllowed("foo.example.com")); + Assert.assertTrue(filter.isOriginAllowed("foo.bar.example.com")); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index c704d3797f7..92c73770996 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -102,6 +102,43 @@ YARN Timeline Server yarn.timeline-service.handler-thread-count 10 + + + Enables cross-origin support (CORS) for web services where + cross-origin web response headers are needed. For example, javascript making + a web services request to the timeline server. + yarn.timeline-service.http-cross-origin.enabled + false + + + + Comma separated list of origins that are allowed for web + services needing cross-origin (CORS) support. Wildcards (*) and patterns + allowed + yarn.timeline-service.http-cross-origin.allowed-origins + * + + + + Comma separated list of methods that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-methods + GET,POST,HEAD + + + + Comma separated list of headers that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-headers + X-Requested-With,Content-Type,Accept,Origin + + + + The number of seconds a pre-flighted request can be cached + for web services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.max-age + 1800 + +---+ * Generic-data related Configuration